Browse Source

Auto configure TLS for new nodes of new clusters (#77231)

This commit introduces TLS auto-configuration for elasticsearch nodes, during
the first startup. A number of heuristics are performed in order to determine if
the node should get TLS auto-configuration which can also be explicitly
disallowed with the use of xpack.security.autoconfiguration.enabled setting.

This affects archive installations and docker. Packaged installations are
handled in #75144 and #75704 .

Co-authored-by: Ioannis Kakavas <ioannis@elastic.co>
Albert Zaharovits 4 years ago
parent
commit
b257da10aa
31 changed files with 1394 additions and 592 deletions
  1. 23 0
      distribution/src/bin/elasticsearch
  2. 13 0
      distribution/src/bin/elasticsearch-env
  3. 22 0
      distribution/src/bin/elasticsearch.bat
  4. 7 0
      docs/changelog/77231.yaml
  5. 3 1
      libs/cli/src/main/java/org/elasticsearch/cli/ExitCodes.java
  6. 1 12
      qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveGenerateInitialCredentialsTests.java
  7. 201 38
      qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java
  8. 17 2
      qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java
  9. 15 3
      qa/os/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java
  10. 49 33
      qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
  11. 71 77
      qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java
  12. 2 5
      qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java
  13. 24 9
      qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java
  14. 197 4
      qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java
  15. 6 4
      qa/os/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java
  16. 2 1
      qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java
  17. 7 8
      qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java
  18. 7 2
      qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java
  19. 0 1
      qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java
  20. 1 1
      qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java
  21. 124 93
      qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java
  22. 6 1
      qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java
  23. 48 11
      qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java
  24. 1 0
      x-pack/plugin/security/cli/build.gradle
  25. 1 0
      x-pack/plugin/security/cli/licenses/commons-io-2.5.jar.sha1
  26. 202 0
      x-pack/plugin/security/cli/licenses/commons-io-LICENSE.txt
  27. 5 0
      x-pack/plugin/security/cli/licenses/commons-io-NOTICE.txt
  28. 336 249
      x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/ConfigInitialNode.java
  29. 3 4
      x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/EnrollNodeToCluster.java
  30. 0 12
      x-pack/plugin/security/src/main/bin/elasticsearch-security-config
  31. 0 21
      x-pack/plugin/security/src/main/bin/elasticsearch-security-config.bat

+ 23 - 0
distribution/src/bin/elasticsearch

@@ -16,11 +16,13 @@
 source "`dirname "$0"`"/elasticsearch-env
 
 CHECK_KEYSTORE=true
+ATTEMPT_SECURITY_AUTO_CONFIG=true
 DAEMONIZE=false
 for option in "$@"; do
   case "$option" in
     -h|--help|-V|--version)
       CHECK_KEYSTORE=false
+      ATTEMPT_SECURITY_AUTO_CONFIG=false
       ;;
     -d|--daemonize)
       DAEMONIZE=true
@@ -45,6 +47,27 @@ then
   fi
 fi
 
+if [[ $ATTEMPT_SECURITY_AUTO_CONFIG = true ]]; then
+  # It is possible that an auto-conf failure prevents the node from starting, but this is only the exceptional case (exit code 1).
+  # Most likely an auto-conf failure will leave the configuration untouched (exit codes 73, 78 and 80), optionally printing a message
+  # if the error is uncommon or unexpected, but it should otherwise let the node to start as usual.
+  # It is passed in all the command line options in order to read the node settings ones (-E), while the other parameters are ignored
+  # (a small caveat is that it also inspects the -v option in order to provide more information on how auto config went)
+  if ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.ConfigInitialNode \
+    ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
+    ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
+    "`dirname "$0"`"/elasticsearch-cli "$@" <<<"$KEYSTORE_PASSWORD"; then
+      :
+  else
+    retval=$?
+    # these exit codes cover the cases where auto-conf cannot run but the node should NOT be prevented from starting as usual
+    # eg the node is restarted, is already configured in an incompatible way, or the file system permissions do not allow it
+    if [[ $retval -ne 80 ]] && [[ $retval -ne 73 ]] && [[ $retval -ne 78 ]]; then
+      exit $retval
+    fi
+  fi
+fi
+
 # The JVM options parser produces the final JVM options to start Elasticsearch.
 # It does this by incorporating JVM options in the following way:
 #   - first, system JVM options are applied (these are hardcoded options in the

+ 13 - 0
distribution/src/bin/elasticsearch-env

@@ -119,11 +119,21 @@ if [[ "$ES_DISTRIBUTION_TYPE" == "docker" ]]; then
 
   declare -a es_arg_array
 
+  containsElement () {
+    local e match="$1"
+    shift
+    for e; do [[ "$e" == "$match" ]] && return 0; done
+    return 1
+  }
+
   # Elasticsearch settings need to either:
   # a. have at least two dot separated lower case words, e.g. `cluster.name`, or
   while IFS='=' read -r envvar_key envvar_value; do
+    es_opt=""
     if [[ -n "$envvar_value" ]]; then
       es_opt="-E${envvar_key}=${envvar_value}"
+    fi
+    if [[ ! -z "${es_opt}" ]] && ! containsElement "${es_opt}" "$@" ; then
       es_arg_array+=("${es_opt}")
     fi
   done <<< "$(env | grep -E '^[-a-z0-9_]+(\.[-a-z0-9_]+)+=')"
@@ -131,10 +141,13 @@ if [[ "$ES_DISTRIBUTION_TYPE" == "docker" ]]; then
   # b. be upper cased with underscore separators and prefixed with `ES_SETTING_`, e.g. `ES_SETTING_CLUSTER_NAME`.
   #    Underscores in setting names are escaped by writing them as a double-underscore e.g. "__"
   while IFS='=' read -r envvar_key envvar_value; do
+    es_opt=""
     if [[ -n "$envvar_value" ]]; then
       # The long-hand sed `y` command works in any sed variant.
       envvar_key="$(echo "$envvar_key" | sed -e 's/^ES_SETTING_//; s/_/./g ; s/\.\./_/g; y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/' )"
       es_opt="-E${envvar_key}=${envvar_value}"
+    fi
+    if [[ ! -z "${es_opt}" ]] && ! containsElement "${es_opt}" "$@" ; then
       es_arg_array+=("${es_opt}")
     fi
   done <<< "$(env | grep -E '^ES_SETTING(_{1,2}[A-Z]+)+=')"

+ 22 - 0
distribution/src/bin/elasticsearch.bat

@@ -5,6 +5,7 @@ setlocal enableextensions
 
 SET params='%*'
 SET checkpassword=Y
+SET attemptautoconfig=Y
 
 :loop
 FOR /F "usebackq tokens=1* delims= " %%A IN (!params!) DO (
@@ -21,16 +22,20 @@ FOR /F "usebackq tokens=1* delims= " %%A IN (!params!) DO (
 
 	IF "!current!" == "-h" (
 		SET checkpassword=N
+		SET attemptautoconfig=N
 	)
 	IF "!current!" == "--help" (
 		SET checkpassword=N
+		SET attemptautoconfig=N
 	)
 
 	IF "!current!" == "-V" (
 		SET checkpassword=N
+		SET attemptautoconfig=N
 	)
 	IF "!current!" == "--version" (
 		SET checkpassword=N
+		SET attemptautoconfig=N
 	)
 
 	IF "!silent!" == "Y" (
@@ -68,6 +73,23 @@ IF "%checkpassword%"=="Y" (
   )
 )
 
+IF "%attemptautoconfig%"=="Y" (
+    ECHO.!KEYSTORE_PASSWORD!| %JAVA% %ES_JAVA_OPTS% ^
+      -Des.path.home="%ES_HOME%" ^
+      -Des.path.conf="%ES_PATH_CONF%" ^
+      -Des.distribution.flavor="%ES_DISTRIBUTION_FLAVOR%" ^
+      -Des.distribution.type="%ES_DISTRIBUTION_TYPE%" ^
+      -cp "!ES_CLASSPATH!;!ES_HOME!/lib/tools/security-cli/*;!ES_HOME!/modules/x-pack-core/*;!ES_HOME!/modules/x-pack-security/*" "org.elasticsearch.xpack.security.cli.ConfigInitialNode" !newparams!
+    SET SHOULDEXIT=Y
+    IF !ERRORLEVEL! EQU 0 SET SHOULDEXIT=N
+    IF !ERRORLEVEL! EQU 73 SET SHOULDEXIT=N
+    IF !ERRORLEVEL! EQU 78 SET SHOULDEXIT=N
+    IF !ERRORLEVEL! EQU 80 SET SHOULDEXIT=N
+    IF "!SHOULDEXIT!"=="Y" (
+        exit /b !ERRORLEVEL!
+    )
+)
+
 if not defined ES_TMPDIR (
   for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory"`) do set  ES_TMPDIR=%%a
 )

+ 7 - 0
docs/changelog/77231.yaml

@@ -0,0 +1,7 @@
+pr: 77231
+summary: Auto configure TLS for new nodes of new clusters
+area: Security
+type: feature
+issues:
+ - 75144
+ - 75704

+ 3 - 1
libs/cli/src/main/java/org/elasticsearch/cli/ExitCodes.java

@@ -12,8 +12,9 @@ package org.elasticsearch.cli;
  * POSIX exit codes.
  */
 public class ExitCodes {
+    // please be extra careful when changing these as the values might be used in scripts,
+    // usages of which are not tracked by the IDE
     public static final int OK = 0;
-    public static final int NOOP = 63;           // nothing to do
     public static final int USAGE = 64;          // command line usage error
     public static final int DATA_ERROR = 65;     // data format error
     public static final int NO_INPUT = 66;       // cannot open input
@@ -27,6 +28,7 @@ public class ExitCodes {
     public static final int PROTOCOL = 76;       // remote error in protocol
     public static final int NOPERM = 77;         // permission denied
     public static final int CONFIG = 78;         // configuration error
+    public static final int NOOP = 80;           // nothing to do
 
     private ExitCodes() { /* no instance, just constants */ }
 }

+ 1 - 12
qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveGenerateInitialCredentialsTests.java

@@ -8,7 +8,6 @@
 
 package org.elasticsearch.packaging.test;
 
-import org.apache.http.client.fluent.Request;
 import org.elasticsearch.packaging.util.Distribution;
 import org.elasticsearch.packaging.util.FileUtils;
 import org.elasticsearch.packaging.util.ServerUtils;
@@ -45,15 +44,10 @@ public class ArchiveGenerateInitialCredentialsTests extends PackagingTestCase {
     }
 
     public void test10Install() throws Exception {
-        // security config tool would run as administrator and change the owner of the config file, which is elasticsearch
-        // We can re-enable this when #77231 is merged, but the rest of the tests in class are also currently muted on windows
-        assumeTrue("Don't run on windows", distribution.platform != Distribution.Platform.WINDOWS);
         installation = installArchive(sh, distribution());
         // Enable security for these tests only where it is necessary, until we can enable it for all
-        // TODO: Remove this when https://github.com/elastic/elasticsearch/pull/77231 is merged
         ServerUtils.enableSecurityFeatures(installation);
         verifyArchiveInstallation(installation, distribution());
-        installation.executables().securityConfigTool.run("");
     }
 
     public void test20NoAutoGenerationWhenAutoConfigurationDisabled() throws Exception {
@@ -88,12 +82,7 @@ public class ArchiveGenerateInitialCredentialsTests extends PackagingTestCase {
         assertThat(parseElasticPassword(result.stdout), notNullValue());
         assertThat(parseKibanaToken(result.stdout), notNullValue());
         assertThat(parseFingerprint(result.stdout), notNullValue());
-        String response = ServerUtils.makeRequest(
-            Request.Get("https://localhost:9200"),
-            "elastic",
-            parseElasticPassword(result.stdout),
-            ServerUtils.getCaCert(installation.config)
-        );
+        String response = makeRequestAsElastic("https://localhost:9200", parseElasticPassword(result.stdout));
         assertThat(response, containsString("You Know, for Search"));
     }
 

+ 201 - 38
qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -9,10 +9,12 @@
 package org.elasticsearch.packaging.test;
 
 import org.apache.http.client.fluent.Request;
+import org.elasticsearch.packaging.util.Distribution;
 import org.elasticsearch.packaging.util.FileUtils;
 import org.elasticsearch.packaging.util.Installation;
 import org.elasticsearch.packaging.util.Platforms;
 import org.elasticsearch.packaging.util.ServerUtils;
+import org.elasticsearch.packaging.util.Shell;
 import org.elasticsearch.packaging.util.Shell.Result;
 import org.junit.BeforeClass;
 
@@ -20,9 +22,13 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
+import java.util.Locale;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static java.nio.file.StandardOpenOption.APPEND;
 import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
 import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER;
 import static org.elasticsearch.packaging.util.Archives.installArchive;
 import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation;
@@ -30,7 +36,6 @@ import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileExists;
 import static org.elasticsearch.packaging.util.FileUtils.append;
 import static org.elasticsearch.packaging.util.FileUtils.mv;
 import static org.elasticsearch.packaging.util.FileUtils.rm;
-import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.is;
@@ -50,6 +55,23 @@ public class ArchiveTests extends PackagingTestCase {
     public void test10Install() throws Exception {
         installation = installArchive(sh, distribution());
         verifyArchiveInstallation(installation, distribution());
+        setFileSuperuser("test_superuser", "test_superuser_password");
+        // See https://bugs.openjdk.java.net/browse/JDK-8267701. In short, when generating PKCS#12 keystores in JDK 12 and later
+        // the MAC algorithm used for integrity protection is incompatible with any previous JDK version. This affects us as we generate
+        // PKCS12 keystores on startup ( with the bundled JDK ) but we also need to run certain tests with a JDK other than the bundled
+        // one, and we still use JDK11 for that.
+        // We're manually setting the HMAC algorithm to something that is compatible with previous versions here. Moving forward, when
+        // min compat JDK is JDK17, we can remove this hack and use the standard security properties file.
+        final Path jdkSecurityProperties = installation.bundledJdk.resolve("conf").resolve("security").resolve("java.security");
+        List<String> lines;
+        try (Stream<String> allLines = Files.readAllLines(jdkSecurityProperties).stream()) {
+            lines = allLines.filter(s -> s.startsWith("#keystore.pkcs12.macAlgorithm") == false)
+                .filter(s -> s.startsWith("#keystore.pkcs12.macIterationCount") == false)
+                .collect(Collectors.toList());
+        }
+        lines.add("keystore.pkcs12.macAlgorithm = HmacPBESHA1");
+        lines.add("keystore.pkcs12.macIterationCount = 100000");
+        Files.write(jdkSecurityProperties, lines, TRUNCATE_EXISTING);
     }
 
     public void test20PluginsListWithNoPlugins() throws Exception {
@@ -70,7 +92,7 @@ public class ArchiveTests extends PackagingTestCase {
                 mv(installation.bundledJdk, relocatedJdk);
             }
             // ask for elasticsearch version to quickly exit if java is actually found (ie test failure)
-            final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString() + " -v");
+            final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString() + " -V");
             assertThat(runResult.exitCode, is(1));
             assertThat(runResult.stderr, containsString("could not find java in bundled JDK"));
         } finally {
@@ -107,28 +129,170 @@ public class ArchiveTests extends PackagingTestCase {
         }
     }
 
-    public void test50StartAndStop() throws Exception {
-        // cleanup from previous test
-        rm(installation.config("elasticsearch.keystore"));
+    public void test40AutoconfigurationNotTriggeredWhenNodeIsMeantToJoinExistingCluster() throws Exception {
+        // auto-config requires that the archive owner and the process user be the same,
+        Platforms.onWindows(() -> sh.chown(installation.config, installation.getOwner()));
+        FileUtils.assertPathsDoNotExist(installation.data);
+        ServerUtils.addSettingToExistingConfiguration(installation, "discovery.seed_hosts", "[\"127.0.0.1:9300\"]");
+        startElasticsearch();
+        verifySecurityNotAutoConfigured(installation);
+        stopElasticsearch();
+        ServerUtils.removeSettingFromExistingConfiguration(installation, "discovery.seed_hosts");
+        Platforms.onWindows(() -> sh.chown(installation.config));
+        FileUtils.rm(installation.data);
+    }
+
+    public void test41AutoconfigurationNotTriggeredWhenNodeCannotContainData() throws Exception {
+        // auto-config requires that the archive owner and the process user be the same
+        Platforms.onWindows(() -> sh.chown(installation.config, installation.getOwner()));
+        ServerUtils.addSettingToExistingConfiguration(installation, "node.roles", "[\"voting_only\", \"master\"]");
+        startElasticsearch();
+        verifySecurityNotAutoConfigured(installation);
+        stopElasticsearch();
+        ServerUtils.removeSettingFromExistingConfiguration(installation, "node.roles");
+        Platforms.onWindows(() -> sh.chown(installation.config));
+        FileUtils.rm(installation.data);
+    }
+
+    public void test42AutoconfigurationNotTriggeredWhenNodeCannotBecomeMaster() throws Exception {
+        // auto-config requires that the archive owner and the process user be the same
+        Platforms.onWindows(() -> sh.chown(installation.config, installation.getOwner()));
+        ServerUtils.addSettingToExistingConfiguration(installation, "node.roles", "[\"ingest\"]");
+        startElasticsearch();
+        verifySecurityNotAutoConfigured(installation);
+        stopElasticsearch();
+        ServerUtils.removeSettingFromExistingConfiguration(installation, "node.roles");
+        Platforms.onWindows(() -> sh.chown(installation.config));
+        FileUtils.rm(installation.data);
+    }
+
+    public void test43AutoconfigurationNotTriggeredWhenTlsAlreadyConfigured() throws Exception {
+        // auto-config requires that the archive owner and the process user be the same
+        Platforms.onWindows(() -> sh.chown(installation.config, installation.getOwner()));
+        ServerUtils.addSettingToExistingConfiguration(installation, "xpack.security.http.ssl.enabled", "false");
+        startElasticsearch();
+        verifySecurityNotAutoConfigured(installation);
+        stopElasticsearch();
+        ServerUtils.removeSettingFromExistingConfiguration(installation, "xpack.security.http.ssl.enabled");
+        Platforms.onWindows(() -> sh.chown(installation.config));
+        FileUtils.rm(installation.data);
+    }
 
+    public void test44AutoConfigurationNotTriggeredOnNotWriteableConfDir() throws Exception {
+        Platforms.onWindows(() -> {
+            // auto-config requires that the archive owner and the process user be the same
+            sh.chown(installation.config, installation.getOwner());
+            // prevent modifications to the config directory
+            sh.run(
+                String.format(
+                    Locale.ROOT,
+                    "$ACL = Get-ACL -Path '%s'; "
+                        + "$AccessRule = New-Object System.Security.AccessControl.FileSystemAccessRule('%s','Write','Deny'); "
+                        + "$ACL.SetAccessRule($AccessRule); "
+                        + "$ACL | Set-Acl -Path '%s';",
+                    installation.config,
+                    installation.getOwner(),
+                    installation.config
+                )
+            );
+        });
+        Platforms.onLinux(() -> { sh.run("chmod u-w " + installation.config); });
         try {
             startElasticsearch();
-        } catch (Exception e) {
-            if (Files.exists(installation.home.resolve("elasticsearch.pid"))) {
-                String pid = FileUtils.slurp(installation.home.resolve("elasticsearch.pid")).trim();
-                logger.info("Dumping jstack of elasticsearch process ({}) that failed to start", pid);
-                sh.runIgnoreExitCode("jstack " + pid);
-            }
-            throw e;
+            verifySecurityNotAutoConfigured(installation);
+            // the node still starts, with Security enabled, but without TLS auto-configured (so only authentication)
+            runElasticsearchTests();
+            stopElasticsearch();
+        } finally {
+            Platforms.onWindows(() -> {
+                sh.run(
+                    String.format(
+                        Locale.ROOT,
+                        "$ACL = Get-ACL -Path '%s'; "
+                            + "$AccessRule = New-Object System.Security.AccessControl.FileSystemAccessRule('%s','Write','Deny'); "
+                            + "$ACL.RemoveAccessRule($AccessRule); "
+                            + "$ACL | Set-Acl -Path '%s';",
+                        installation.config,
+                        installation.getOwner(),
+                        installation.config
+                    )
+                );
+                sh.chown(installation.config);
+            });
+            Platforms.onLinux(() -> { sh.run("chmod u+w " + installation.config); });
+            FileUtils.rm(installation.data);
         }
+    }
 
-        assertThat(installation.logs.resolve("gc.log"), fileExists());
-        ServerUtils.runElasticsearchTests();
+    public void test50AutoConfigurationFailsWhenCertificatesNotGenerated() throws Exception {
+        // auto-config requires that the archive owner and the process user be the same
+        Platforms.onWindows(() -> sh.chown(installation.config, installation.getOwner()));
+        FileUtils.assertPathsDoNotExist(installation.data);
+        Path tempDir = createTempDir("bc-backup");
+        Files.move(
+            installation.lib.resolve("tools").resolve("security-cli").resolve("bcprov-jdk15on-1.64.jar"),
+            tempDir.resolve("bcprov-jdk15on-1.64.jar")
+        );
+        Shell.Result result = runElasticsearchStartCommand(null, false, false);
+        assertElasticsearchFailure(result, "java.lang.NoClassDefFoundError: org/bouncycastle/asn1/x509/GeneralName", null);
+        Files.move(
+            tempDir.resolve("bcprov-jdk15on-1.64.jar"),
+            installation.lib.resolve("tools").resolve("security-cli").resolve("bcprov-jdk15on-1.64.jar")
+        );
+        Platforms.onWindows(() -> sh.chown(installation.config));
+        FileUtils.rm(tempDir);
+    }
+
+    public void test51AutoConfigurationWithPasswordProtectedKeystore() throws Exception {
+        /* Windows issue awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
+        assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);
+        FileUtils.assertPathsDoNotExist(installation.data);
+        final Installation.Executables bin = installation.executables();
+        final String password = "some-keystore-password";
+        Platforms.onLinux(() -> bin.keystoreTool.run("passwd", password + "\n" + password + "\n"));
+        Platforms.onWindows(
+            () -> {
+                sh.run("Invoke-Command -ScriptBlock {echo '" + password + "'; echo '" + password + "'} | " + bin.keystoreTool + " passwd");
+            }
+        );
+        Shell.Result result = runElasticsearchStartCommand("some-wrong-password-here", false, false);
+        assertElasticsearchFailure(result, "Provided keystore password was incorrect", null);
+        verifySecurityNotAutoConfigured(installation);
+
+        awaitElasticsearchStartup(runElasticsearchStartCommand(password, true, true));
+        verifySecurityAutoConfigured(installation);
 
         stopElasticsearch();
+
+        // Revert to an empty password for the rest of the tests
+        Platforms.onLinux(() -> bin.keystoreTool.run("passwd", password + "\n" + "" + "\n"));
+        Platforms.onWindows(
+            () -> sh.run("Invoke-Command -ScriptBlock {echo '" + password + "'; echo '" + "" + "'} | " + bin.keystoreTool + " passwd")
+        );
+    }
+
+    public void test52AutoConfigurationOnWindows() throws Exception {
+        assumeTrue(
+            "run this in place of test51AutoConfigurationWithPasswordProtectedKeystore on windows",
+            distribution.platform == Distribution.Platform.WINDOWS
+        );
+        sh.chown(installation.config, installation.getOwner());
+        FileUtils.assertPathsDoNotExist(installation.data);
+
+        startElasticsearch();
+        verifySecurityAutoConfigured(installation);
+        stopElasticsearch();
+        sh.chown(installation.config);
     }
 
-    public void test51EsJavaHomeOverride() throws Exception {
+    public void test60StartAndStop() throws Exception {
+        startElasticsearch();
+        assertThat(installation.logs.resolve("gc.log"), fileExists());
+        runElasticsearchTests();
+        stopElasticsearch();
+    }
+
+    public void test61EsJavaHomeOverride() throws Exception {
         Platforms.onLinux(() -> {
             String systemJavaHome1 = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
             sh.getEnv().put("ES_JAVA_HOME", systemJavaHome1);
@@ -139,14 +303,14 @@ public class ArchiveTests extends PackagingTestCase {
         });
 
         startElasticsearch();
-        ServerUtils.runElasticsearchTests();
+        runElasticsearchTests();
         stopElasticsearch();
 
         String systemJavaHome1 = sh.getEnv().get("ES_JAVA_HOME");
         assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"), containsString(systemJavaHome1));
     }
 
-    public void test51JavaHomeIgnored() throws Exception {
+    public void test62JavaHomeIgnored() throws Exception {
         assumeTrue(distribution().hasJdk);
         Platforms.onLinux(() -> {
             String systemJavaHome1 = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
@@ -166,7 +330,7 @@ public class ArchiveTests extends PackagingTestCase {
         assertThat(runResult.stderr, containsString("warning: ignoring JAVA_HOME=" + systemJavaHome + "; using bundled JDK"));
 
         startElasticsearch();
-        ServerUtils.runElasticsearchTests();
+        runElasticsearchTests();
         stopElasticsearch();
 
         // if the JDK started with the bundled JDK then we know that JAVA_HOME was ignored
@@ -174,7 +338,7 @@ public class ArchiveTests extends PackagingTestCase {
         assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"), containsString(bundledJdk));
     }
 
-    public void test52BundledJdkRemoved() throws Exception {
+    public void test63BundledJdkRemoved() throws Exception {
         assumeThat(distribution().hasJdk, is(true));
 
         Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
@@ -190,7 +354,7 @@ public class ArchiveTests extends PackagingTestCase {
             });
 
             startElasticsearch();
-            ServerUtils.runElasticsearchTests();
+            runElasticsearchTests();
             stopElasticsearch();
 
             String systemJavaHome1 = sh.getEnv().get("ES_JAVA_HOME");
@@ -200,7 +364,7 @@ public class ArchiveTests extends PackagingTestCase {
         }
     }
 
-    public void test53JavaHomeWithSpecialCharacters() throws Exception {
+    public void test64JavaHomeWithSpecialCharacters() throws Exception {
         Platforms.onWindows(() -> {
             String javaPath = "C:\\Program Files (x86)\\java";
             try {
@@ -211,7 +375,7 @@ public class ArchiveTests extends PackagingTestCase {
 
                 // verify ES can start, stop and run plugin list
                 startElasticsearch();
-
+                runElasticsearchTests();
                 stopElasticsearch();
 
                 String pluginListCommand = installation.bin + "/elasticsearch-plugin list";
@@ -236,7 +400,7 @@ public class ArchiveTests extends PackagingTestCase {
 
                 // verify ES can start, stop and run plugin list
                 startElasticsearch();
-
+                runElasticsearchTests();
                 stopElasticsearch();
 
                 String pluginListCommand = installation.bin + "/elasticsearch-plugin list";
@@ -248,15 +412,13 @@ public class ArchiveTests extends PackagingTestCase {
         });
     }
 
-    public void test54ForceBundledJdkEmptyJavaHome() throws Exception {
+    public void test65ForceBundledJdkEmptyJavaHome() throws Exception {
         assumeThat(distribution().hasJdk, is(true));
-        // cleanup from previous test
-        rm(installation.config("elasticsearch.keystore"));
 
         sh.getEnv().put("ES_JAVA_HOME", "");
 
         startElasticsearch();
-        ServerUtils.runElasticsearchTests();
+        runElasticsearchTests();
         stopElasticsearch();
     }
 
@@ -265,25 +427,28 @@ public class ArchiveTests extends PackagingTestCase {
      * <p>
      * This test purposefully ignores the existence of the Windows POSIX sub-system.
      */
-    public void test55InstallUnderPosix() throws Exception {
-        assumeTrue("Only run this test on Unix-like systems", Platforms.WINDOWS == false);
+    public void test66InstallUnderPosix() throws Exception {
         sh.getEnv().put("POSIXLY_CORRECT", "1");
         startElasticsearch();
+        runElasticsearchTests();
         stopElasticsearch();
     }
 
     public void test70CustomPathConfAndJvmOptions() throws Exception {
-
         withCustomConfig(tempConf -> {
             setHeap("512m", tempConf);
             final List<String> jvmOptions = List.of("-Dlog4j2.disable.jmx=true");
             Files.write(tempConf.resolve("jvm.options"), jvmOptions, CREATE, APPEND);
 
             sh.getEnv().put("ES_JAVA_OPTS", "-XX:-UseCompressedOops");
-
             startElasticsearch();
 
-            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            final String nodesResponse = ServerUtils.makeRequest(
+                Request.Get("https://localhost:9200/_nodes"),
+                "test_superuser",
+                "test_superuser_password",
+                ServerUtils.getCaCert(tempConf)
+            );
             assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
             assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
 
@@ -299,7 +464,7 @@ public class ArchiveTests extends PackagingTestCase {
 
             startElasticsearch();
 
-            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            final String nodesResponse = makeRequest("https://localhost:9200/_nodes");
             assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
 
             stopElasticsearch();
@@ -322,7 +487,7 @@ public class ArchiveTests extends PackagingTestCase {
 
             startElasticsearch();
 
-            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            final String nodesResponse = makeRequest("https://localhost:9200/_nodes");
             assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
             assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
 
@@ -339,7 +504,7 @@ public class ArchiveTests extends PackagingTestCase {
             append(jvmOptionsIgnored, "-Xthis_is_not_a_valid_option\n");
 
             startElasticsearch();
-            ServerUtils.runElasticsearchTests();
+            runElasticsearchTests();
             stopElasticsearch();
         } finally {
             rm(jvmOptionsIgnored);
@@ -347,13 +512,11 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test80RelativePathConf() throws Exception {
-
         withCustomConfig(tempConf -> {
             append(tempConf.resolve("elasticsearch.yml"), "node.name: relative");
-
             startElasticsearch();
 
-            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            final String nodesResponse = makeRequest("https://localhost:9200/_nodes");
             assertThat(nodesResponse, containsString("\"name\":\"relative\""));
 
             stopElasticsearch();

+ 17 - 2
qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java

@@ -20,10 +20,12 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeFalse;
 import static java.nio.file.StandardOpenOption.APPEND;
 import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
 import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File;
 import static org.elasticsearch.packaging.util.FileMatcher.file;
 import static org.elasticsearch.packaging.util.FileMatcher.p600;
@@ -47,7 +49,10 @@ public class CertGenCliTests extends PackagingTestCase {
     public void test10Install() throws Exception {
         install();
         // Enable security for this test only where it is necessary, until we can enable it for all
+        // Only needed until https://github.com/elastic/elasticsearch/pull/75144 is merged
         ServerUtils.enableSecurityFeatures(installation);
+        // Disable security auto-configuration as we want to generate keys/certificates manually here
+        ServerUtils.disableSecurityAutoConfiguration(installation);
     }
 
     public void test20Help() {
@@ -95,7 +100,7 @@ public class CertGenCliTests extends PackagingTestCase {
         final String certPath = escapePath(installation.config("certs/mynode/mynode.crt"));
         final String caCertPath = escapePath(installation.config("certs/ca/ca.crt"));
 
-        List<String> yaml = List.of(
+        final List<String> tlsConfig = List.of(
             "node.name: mynode",
             "xpack.security.transport.ssl.key: " + keyPath,
             "xpack.security.transport.ssl.certificate: " + certPath,
@@ -107,7 +112,17 @@ public class CertGenCliTests extends PackagingTestCase {
             "xpack.security.http.ssl.enabled: true"
         );
 
-        Files.write(installation.config("elasticsearch.yml"), yaml, CREATE, APPEND);
+        // TODO: Simplify this when https://github.com/elastic/elasticsearch/pull/75144 is merged. We only need to
+        // filter settings from the existing config as they are explicitly set to false on package installation
+        List<String> existingConfig = Files.readAllLines(installation.config("elasticsearch.yml"));
+        List<String> newConfig = existingConfig.stream()
+            .filter(l -> l.startsWith("node.name:") == false)
+            .filter(l -> l.startsWith("xpack.security.transport.ssl.") == false)
+            .filter(l -> l.startsWith("xpack.security.http.ssl.") == false)
+            .collect(Collectors.toList());
+        newConfig.addAll(tlsConfig);
+
+        Files.write(installation.config("elasticsearch.yml"), newConfig, TRUNCATE_EXISTING);
 
         assertWhileRunning(() -> {
             final String password = setElasticPassword();

+ 15 - 3
qa/os/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java

@@ -11,10 +11,10 @@ package org.elasticsearch.packaging.test;
 import org.apache.http.client.fluent.Request;
 import org.elasticsearch.packaging.util.FileUtils;
 import org.elasticsearch.packaging.util.Platforms;
+import org.elasticsearch.packaging.util.ServerUtils;
 import org.junit.Before;
 
 import static org.elasticsearch.packaging.util.FileUtils.append;
-import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assume.assumeFalse;
 
@@ -27,9 +27,10 @@ public class ConfigurationTests extends PackagingTestCase {
 
     public void test10Install() throws Exception {
         install();
+        setFileSuperuser("test_superuser", "test_superuser_password");
     }
 
-    public void test60HostnameSubstitution() throws Exception {
+    public void test20HostnameSubstitution() throws Exception {
         String hostnameKey = Platforms.WINDOWS ? "COMPUTERNAME" : "HOSTNAME";
         sh.getEnv().put(hostnameKey, "mytesthost");
         withCustomConfig(confPath -> {
@@ -37,10 +38,21 @@ public class ConfigurationTests extends PackagingTestCase {
             if (distribution.isPackage()) {
                 append(installation.envFile, "HOSTNAME=mytesthost");
             }
+            // Packaged installations don't get autoconfigured yet
+            // TODO: Remove this in https://github.com/elastic/elasticsearch/pull/75144
+            String protocol = distribution.isPackage() ? "http" : "https";
+            // security auto-config requires that the archive owner and the node process user be the same
+            Platforms.onWindows(() -> sh.chown(confPath, installation.getOwner()));
             assertWhileRunning(() -> {
-                final String nameResponse = makeRequest(Request.Get("http://localhost:9200/_cat/nodes?h=name")).strip();
+                final String nameResponse = ServerUtils.makeRequest(
+                    Request.Get(protocol + "://localhost:9200/_cat/nodes?h=name"),
+                    "test_superuser",
+                    "test_superuser_password",
+                    ServerUtils.getCaCert(confPath)
+                ).strip();
                 assertThat(nameResponse, equalTo("mytesthost"));
             });
+            Platforms.onWindows(() -> sh.chown(confPath));
         });
     }
 }

+ 49 - 33
qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -42,6 +42,7 @@ import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File;
 import static org.elasticsearch.packaging.util.FileMatcher.p600;
 import static org.elasticsearch.packaging.util.FileMatcher.p644;
 import static org.elasticsearch.packaging.util.FileMatcher.p660;
+import static org.elasticsearch.packaging.util.FileMatcher.p750;
 import static org.elasticsearch.packaging.util.FileMatcher.p755;
 import static org.elasticsearch.packaging.util.FileMatcher.p775;
 import static org.elasticsearch.packaging.util.FileUtils.append;
@@ -49,6 +50,7 @@ import static org.elasticsearch.packaging.util.FileUtils.rm;
 import static org.elasticsearch.packaging.util.docker.Docker.chownWithPrivilegeEscalation;
 import static org.elasticsearch.packaging.util.docker.Docker.copyFromContainer;
 import static org.elasticsearch.packaging.util.docker.Docker.existsInContainer;
+import static org.elasticsearch.packaging.util.docker.Docker.findInContainer;
 import static org.elasticsearch.packaging.util.docker.Docker.getContainerLogs;
 import static org.elasticsearch.packaging.util.docker.Docker.getImageHealthcheck;
 import static org.elasticsearch.packaging.util.docker.Docker.getImageLabels;
@@ -91,7 +93,6 @@ import static org.junit.Assume.assumeTrue;
  */
 public class DockerTests extends PackagingTestCase {
     private Path tempDir;
-    private static final String USERNAME = "elastic";
     private static final String PASSWORD = "nothunter2";
 
     @BeforeClass
@@ -114,16 +115,17 @@ public class DockerTests extends PackagingTestCase {
     /**
      * Checks that the Docker image can be run, and that it passes various checks.
      */
-    public void test010Install() {
+    public void test010Install() throws Exception {
         verifyContainerInstallation(installation);
+        verifySecurityAutoConfigured(installation);
     }
 
     /**
      * Check that security is enabled
      */
     public void test011SecurityEnabledStatus() throws Exception {
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
-        final int statusCode = ServerUtils.makeRequestAndGetStatus(Request.Get("http://localhost:9200"), USERNAME, "wrong_password", null);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
+        final int statusCode = makeRequestAsElastic("wrong_password");
         assertThat(statusCode, equalTo(401));
     }
 
@@ -218,7 +220,7 @@ public class DockerTests extends PackagingTestCase {
      * Check that when the keystore is created on startup, it is created with the correct permissions.
      */
     public void test042KeystorePermissionsAreCorrect() {
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
 
         assertThat(installation.config("elasticsearch.keystore"), file(p660));
     }
@@ -228,11 +230,11 @@ public class DockerTests extends PackagingTestCase {
      * is minimally functional.
      */
     public void test050BasicApiTests() throws Exception {
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
 
         assertTrue(existsInContainer(installation.logs.resolve("gc.log")));
 
-        ServerUtils.runElasticsearchTests(USERNAME, PASSWORD);
+        runElasticsearchTestsAsElastic(PASSWORD);
     }
 
     /**
@@ -240,7 +242,11 @@ public class DockerTests extends PackagingTestCase {
      */
     public void test070BindMountCustomPathConfAndJvmOptions() throws Exception {
         copyFromContainer(installation.config("elasticsearch.yml"), tempDir.resolve("elasticsearch.yml"));
+        copyFromContainer(installation.config("elasticsearch.keystore"), tempDir.resolve("elasticsearch.keystore"));
         copyFromContainer(installation.config("log4j2.properties"), tempDir.resolve("log4j2.properties"));
+        final Path autoConfigurationDir = findInContainer(installation.config, "d", "\"tls_auto_config_initial_node_*\"");
+        final String autoConfigurationDirName = autoConfigurationDir.getFileName().toString();
+        copyFromContainer(autoConfigurationDir, tempDir.resolve(autoConfigurationDirName));
 
         // we have to disable Log4j from using JMX lest it will hit a security
         // manager exception before we have configured logging; this will fail
@@ -252,7 +258,9 @@ public class DockerTests extends PackagingTestCase {
         Files.setPosixFilePermissions(tempDir, fromString("rwxrwxrwx"));
         // These permissions are necessary to run the tests under Vagrant
         Files.setPosixFilePermissions(tempDir.resolve("elasticsearch.yml"), p644);
+        Files.setPosixFilePermissions(tempDir.resolve("elasticsearch.keystore"), p644);
         Files.setPosixFilePermissions(tempDir.resolve("log4j2.properties"), p644);
+        Files.setPosixFilePermissions(tempDir.resolve(autoConfigurationDirName), p750);
 
         // Restart the container
         runContainer(
@@ -262,9 +270,9 @@ public class DockerTests extends PackagingTestCase {
                 .envVar("ELASTIC_PASSWORD", PASSWORD)
         );
 
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
 
-        final JsonNode nodes = getJson("/_nodes", USERNAME, PASSWORD).get("nodes");
+        final JsonNode nodes = getJson("/_nodes", "elastic", PASSWORD, ServerUtils.getCaCert(installation)).get("nodes");
         final String nodeId = nodes.fieldNames().next();
 
         final int heapSize = nodes.at("/" + nodeId + "/jvm/mem/heap_init_in_bytes").intValue();
@@ -290,10 +298,9 @@ public class DockerTests extends PackagingTestCase {
                 distribution(),
                 builder().volume(tempEsDataDir.toAbsolutePath(), installation.data).envVar("ELASTIC_PASSWORD", PASSWORD)
             );
+            waitForElasticsearch(installation, "elastic", PASSWORD);
 
-            waitForElasticsearch(installation, USERNAME, PASSWORD);
-
-            final JsonNode nodes = getJson("/_nodes", USERNAME, PASSWORD);
+            final JsonNode nodes = getJson("/_nodes", "elastic", PASSWORD, ServerUtils.getCaCert(installation));
 
             assertThat(nodes.at("/_nodes/total").intValue(), equalTo(1));
             assertThat(nodes.at("/_nodes/successful").intValue(), equalTo(1));
@@ -327,7 +334,11 @@ public class DockerTests extends PackagingTestCase {
 
         copyFromContainer(installation.config("elasticsearch.yml"), tempEsConfigDir);
         copyFromContainer(installation.config("jvm.options"), tempEsConfigDir);
+        copyFromContainer(installation.config("elasticsearch.keystore"), tempEsConfigDir);
         copyFromContainer(installation.config("log4j2.properties"), tempEsConfigDir);
+        final Path autoConfigurationDir = findInContainer(installation.config, "d", "\"tls_auto_config_initial_node_*\"");
+        final String autoConfigurationDirName = autoConfigurationDir.getFileName().toString();
+        copyFromContainer(autoConfigurationDir, tempEsConfigDir.resolve(autoConfigurationDirName));
 
         chownWithPrivilegeEscalation(tempEsConfigDir, "501:501");
         chownWithPrivilegeEscalation(tempEsDataDir, "501:501");
@@ -343,7 +354,10 @@ public class DockerTests extends PackagingTestCase {
                 .volume(tempEsLogsDir.toAbsolutePath(), installation.logs)
         );
 
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
+        rmDirWithPrivilegeEscalation(tempEsConfigDir);
+        rmDirWithPrivilegeEscalation(tempEsDataDir);
+        rmDirWithPrivilegeEscalation(tempEsLogsDir);
     }
 
     /**
@@ -354,7 +368,7 @@ public class DockerTests extends PackagingTestCase {
         // Restart the container
         runContainer(distribution(), builder().extraArgs("--group-add 0").uid(501, 501).envVar("ELASTIC_PASSWORD", PASSWORD));
 
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
     }
 
     /**
@@ -381,7 +395,7 @@ public class DockerTests extends PackagingTestCase {
 
         // If we configured security correctly, then this call will only work if we specify the correct credentials.
         try {
-            waitForElasticsearch("green", null, installation, "elastic", "hunter2");
+            waitForElasticsearch(installation, "elastic", "hunter2");
         } catch (Exception e) {
             throw new AssertionError(
                 "Failed to check whether Elasticsearch had started. This could be because "
@@ -391,7 +405,12 @@ public class DockerTests extends PackagingTestCase {
         }
 
         // Also check that an unauthenticated call fails
-        final int statusCode = Request.Get("http://localhost:9200/_nodes").execute().returnResponse().getStatusLine().getStatusCode();
+        final int statusCode = ServerUtils.makeRequestAndGetStatus(
+            Request.Get("https://localhost:9200"),
+            null,
+            null,
+            ServerUtils.getCaCert(installation)
+        );
         assertThat("Expected server to require authentication", statusCode, equalTo(401));
     }
 
@@ -526,7 +545,7 @@ public class DockerTests extends PackagingTestCase {
         installation = runContainer(distribution(), builder().envVar("ELASTIC_PASSWORD", "hunter2"));
 
         // The tool below requires a keystore, so ensure that ES is fully initialised before proceeding.
-        waitForElasticsearch("green", null, installation, "elastic", "hunter2");
+        waitForElasticsearch(installation, "elastic", "hunter2");
 
         sh.getEnv().put("http.host", "this.is.not.valid");
 
@@ -753,7 +772,7 @@ public class DockerTests extends PackagingTestCase {
      * Check that the container logs contain the expected content for Elasticsearch itself.
      */
     public void test120DockerLogsIncludeElasticsearchLogs() {
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
         final Result containerLogs = getContainerLogs();
 
         assertThat("Container logs should contain full class names", containerLogs.stdout, containsString("org.elasticsearch.node.Node"));
@@ -766,16 +785,12 @@ public class DockerTests extends PackagingTestCase {
     public void test121CanUseStackLoggingConfig() {
         runContainer(distribution(), builder().envVar("ES_LOG_STYLE", "file").envVar("ELASTIC_PASSWORD", PASSWORD));
 
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
 
         final Result containerLogs = getContainerLogs();
         final List<String> stdout = containerLogs.stdout.lines().collect(Collectors.toList());
-
-        assertThat(
-            "Container logs should be formatted using the stack config",
-            stdout.get(stdout.size() - 1),
-            matchesPattern("^\\[\\d\\d\\d\\d-.*")
-        );
+        // We select to look for a line near the beginning so that we don't stumble upon the stdout printing of auto-configured credentials
+        assertThat("Container logs should be formatted using the stack config", stdout.get(10), matchesPattern("^\\[\\d\\d\\d\\d-.*"));
         assertThat("[logs/docker-cluster.log] should exist but it doesn't", existsInContainer("logs/docker-cluster.log"), is(true));
     }
 
@@ -785,12 +800,12 @@ public class DockerTests extends PackagingTestCase {
     public void test122CanUseDockerLoggingConfig() {
         runContainer(distribution(), builder().envVar("ES_LOG_STYLE", "console").envVar("ELASTIC_PASSWORD", PASSWORD));
 
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
 
         final Result containerLogs = getContainerLogs();
         final List<String> stdout = containerLogs.stdout.lines().collect(Collectors.toList());
-
-        assertThat("Container logs should be formatted using the docker config", stdout.get(stdout.size() - 1), startsWith("{\""));
+        // We select to look for a line near the beginning so that we don't stumble upon the stdout printing of auto-configured credentials
+        assertThat("Container logs should be formatted using the docker config", stdout.get(10), startsWith("{\""));
         assertThat("[logs/docker-cluster.log] shouldn't exist but it does", existsInContainer("logs/docker-cluster.log"), is(false));
     }
 
@@ -809,12 +824,12 @@ public class DockerTests extends PackagingTestCase {
     public void test124CanRestartContainerWithStackLoggingConfig() {
         runContainer(distribution(), builder().envVar("ES_LOG_STYLE", "file").envVar("ELASTIC_PASSWORD", PASSWORD));
 
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
 
         restartContainer();
 
         // If something went wrong running Elasticsearch the second time, this will fail.
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
     }
 
     /**
@@ -850,9 +865,9 @@ public class DockerTests extends PackagingTestCase {
      * Check that Elasticsearch reports per-node cgroup information.
      */
     public void test140CgroupOsStatsAreAvailable() throws Exception {
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", PASSWORD);
 
-        final JsonNode nodes = getJson("/_nodes/stats/os", USERNAME, PASSWORD).get("nodes");
+        final JsonNode nodes = getJson("/_nodes/stats/os", "elastic", PASSWORD, ServerUtils.getCaCert(installation)).get("nodes");
 
         final String nodeId = nodes.fieldNames().next();
 
@@ -885,7 +900,8 @@ public class DockerTests extends PackagingTestCase {
             distribution(),
             builder().memory("942m").volume(jvmOptionsPath, containerJvmOptionsPath).envVar("ELASTIC_PASSWORD", PASSWORD)
         );
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
+
+        waitForElasticsearch(installation, "elastic", PASSWORD);
 
         // Grab the container output and find the line where it print the JVM arguments. This will
         // let us see what the automatic heap sizing calculated.

+ 71 - 77
qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java

@@ -55,8 +55,10 @@ public class KeystoreManagementTests extends PackagingTestCase {
     public static final String ERROR_CORRUPTED_KEYSTORE = "Keystore has been corrupted or tampered with";
     public static final String ERROR_KEYSTORE_NOT_PASSWORD_PROTECTED = "ERROR: Keystore is not password-protected";
     public static final String ERROR_KEYSTORE_NOT_FOUND = "ERROR: Elasticsearch keystore not found";
-    private static final String USERNAME = "elastic";
-    private static final String PASSWORD = "nothunter2";
+    private static final String ELASTIC_PASSWORD = "nothunter2";
+    private static final String FILE_REALM_SUPERUSER = "test-user";
+    private static final String FILE_REALM_SUPERUSER_PASSWORD = "test-user-password";
+    private static final String KEYSTORE_PASSWORD = "keystore-password";
 
     /** Test initial archive state */
     public void test10InstallArchiveDistribution() throws Exception {
@@ -64,6 +66,9 @@ public class KeystoreManagementTests extends PackagingTestCase {
 
         installation = installArchive(sh, distribution);
         verifyArchiveInstallation(installation, distribution());
+        // Add a user for tests to use.
+        // TODO: Possibly capture autoconfigured password from running the node the first time
+        setFileSuperuser(FILE_REALM_SUPERUSER, FILE_REALM_SUPERUSER_PASSWORD);
 
         final Installation.Executables bin = installation.executables();
         Shell.Result r = sh.runIgnoreExitCode(bin.keystoreTool + " has-passwd");
@@ -79,6 +84,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
         installation = installPackage(sh, distribution);
         assertInstalled(distribution);
         verifyPackageInstallation(installation, distribution, sh);
+        // We don't add a user here. We explicitly disable security for packages for now after installation. We will update in a followup
 
         final Installation.Executables bin = installation.executables();
         Shell.Result r = sh.runIgnoreExitCode(bin.keystoreTool + " has-passwd");
@@ -108,50 +114,20 @@ public class KeystoreManagementTests extends PackagingTestCase {
         assertThat(r2.stdout, containsString("keystore.seed"));
     }
 
-    public void test20CreateKeystoreManually() throws Exception {
-        rmKeystoreIfExists();
-        createKeystore(null);
-
-        final Installation.Executables bin = installation.executables();
-        verifyKeystorePermissions();
-
-        Shell.Result r = bin.keystoreTool.run("list");
-        assertThat(r.stdout, containsString("keystore.seed"));
-    }
-
-    public void test30AutoCreateKeystore() throws Exception {
-        assumeTrue("Packages and docker are installed with a keystore file", distribution.isArchive());
-        rmKeystoreIfExists();
-
-        startElasticsearch();
-        stopElasticsearch();
-
-        Platforms.onWindows(() -> sh.chown(installation.config("elasticsearch.keystore")));
-
-        verifyKeystorePermissions();
-
-        final Installation.Executables bin = installation.executables();
-        Shell.Result r = bin.keystoreTool.run("list");
-        assertThat(r.stdout, containsString("keystore.seed"));
-    }
-
-    public void test40KeystorePasswordOnStandardInput() throws Exception {
+    public void test20KeystorePasswordOnStandardInput() throws Exception {
         assumeTrue("packages will use systemd, which doesn't handle stdin", distribution.isArchive());
         assumeThat(installation, is(notNullValue()));
 
-        String password = "^|<>\\&exit"; // code insertion on Windows if special characters are not escaped
-
-        rmKeystoreIfExists();
-        createKeystore(password);
+        createKeystore(KEYSTORE_PASSWORD);
 
         assertPasswordProtectedKeystore();
 
-        awaitElasticsearchStartup(runElasticsearchStartCommand(password, true, false));
-        ServerUtils.runElasticsearchTests();
+        awaitElasticsearchStartup(runElasticsearchStartCommand(KEYSTORE_PASSWORD, true, false));
+        runElasticsearchTests();
         stopElasticsearch();
     }
 
-    public void test41WrongKeystorePasswordOnStandardInput() throws Exception {
+    public void test21WrongKeystorePasswordOnStandardInput() throws Exception {
         assumeTrue("packages will use systemd, which doesn't handle stdin", distribution.isArchive());
         assumeThat(installation, is(notNullValue()));
 
@@ -161,25 +137,20 @@ public class KeystoreManagementTests extends PackagingTestCase {
         assertElasticsearchFailure(result, Arrays.asList(ERROR_INCORRECT_PASSWORD, ERROR_CORRUPTED_KEYSTORE), null);
     }
 
-    public void test42KeystorePasswordOnTty() throws Exception {
+    public void test22KeystorePasswordOnTty() throws Exception {
         /* Windows issue awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
         assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);
         assumeTrue("packages will use systemd, which doesn't handle stdin", distribution.isArchive());
         assumeThat(installation, is(notNullValue()));
 
-        String password = "keystorepass";
-
-        rmKeystoreIfExists();
-        createKeystore(password);
-
         assertPasswordProtectedKeystore();
 
-        awaitElasticsearchStartup(runElasticsearchStartCommand(password, true, true));
-        ServerUtils.runElasticsearchTests();
+        awaitElasticsearchStartup(runElasticsearchStartCommand(KEYSTORE_PASSWORD, true, true));
+        runElasticsearchTests();
         stopElasticsearch();
     }
 
-    public void test43WrongKeystorePasswordOnTty() throws Exception {
+    public void test23WrongKeystorePasswordOnTty() throws Exception {
         /* Windows issue awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
         assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);
         assumeTrue("packages will use systemd, which doesn't handle stdin", distribution.isArchive());
@@ -196,26 +167,20 @@ public class KeystoreManagementTests extends PackagingTestCase {
      * If we have an encrypted keystore, we shouldn't require a password to
      * view help information.
      */
-    public void test44EncryptedKeystoreAllowsHelpMessage() throws Exception {
+    public void test24EncryptedKeystoreAllowsHelpMessage() throws Exception {
         assumeTrue("users call elasticsearch directly in archive case", distribution.isArchive());
 
-        String password = "keystorepass";
-
-        rmKeystoreIfExists();
-        createKeystore(password);
-
         assertPasswordProtectedKeystore();
         Shell.Result r = installation.executables().elasticsearch.run("--help");
         assertThat(r.stdout, startsWith("Starts Elasticsearch"));
     }
 
-    public void test50KeystorePasswordFromFile() throws Exception {
+    public void test30KeystorePasswordFromFile() throws Exception {
         assumeTrue("only for systemd", Platforms.isSystemd() && distribution().isPackage());
-        String password = "!@#$%^&*()|\\<>/?";
         Path esKeystorePassphraseFile = installation.config.resolve("eks");
 
         rmKeystoreIfExists();
-        createKeystore(password);
+        createKeystore(KEYSTORE_PASSWORD);
 
         assertPasswordProtectedKeystore();
 
@@ -223,7 +188,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
             sh.run("sudo systemctl set-environment ES_KEYSTORE_PASSPHRASE_FILE=" + esKeystorePassphraseFile);
 
             Files.createFile(esKeystorePassphraseFile);
-            Files.write(esKeystorePassphraseFile, List.of(password));
+            Files.write(esKeystorePassphraseFile, List.of(KEYSTORE_PASSWORD));
 
             startElasticsearch();
             ServerUtils.runElasticsearchTests();
@@ -233,7 +198,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
         }
     }
 
-    public void test51WrongKeystorePasswordFromFile() throws Exception {
+    public void test31WrongKeystorePasswordFromFile() throws Exception {
         assumeTrue("only for systemd", Platforms.isSystemd() && distribution().isPackage());
         Path esKeystorePassphraseFile = installation.config.resolve("eks");
 
@@ -261,42 +226,40 @@ public class KeystoreManagementTests extends PackagingTestCase {
      * Check that we can mount a password-protected keystore to a docker image
      * and provide a password via an environment variable.
      */
-    @AwaitsFix(bugUrl = "Keystore fails to save with resource busy")
-    public void test60DockerEnvironmentVariablePassword() throws Exception {
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/76124")
+    public void test40DockerEnvironmentVariablePassword() throws Exception {
         assumeTrue(distribution().isDocker());
-        String password = "keystore-password";
 
-        Path localConfigDir = getMountedLocalConfDirWithKeystore(password, installation.config);
+        Path localConfigDir = getMountedLocalConfDirWithKeystore(KEYSTORE_PASSWORD, installation.config);
 
         // restart ES with password and mounted config dir containing password protected keystore
         runContainer(
             distribution(),
             builder().volume(localConfigDir.resolve("config"), installation.config)
-                .envVar("KEYSTORE_PASSWORD", password)
-                .envVar("ELASTIC_PASSWORD", PASSWORD)
+                .envVar("KEYSTORE_PASSWORD", KEYSTORE_PASSWORD)
+                .envVar("ELASTIC_PASSWORD", ELASTIC_PASSWORD)
         );
-        waitForElasticsearch(installation, USERNAME, PASSWORD);
-        ServerUtils.runElasticsearchTests(USERNAME, PASSWORD);
+        waitForElasticsearch(installation, "elastic", ELASTIC_PASSWORD);
+        runElasticsearchTestsAsElastic(ELASTIC_PASSWORD);
     }
 
     /**
      * Check that we can mount a password-protected keystore to a docker image
      * and provide a password via a file, pointed at from an environment variable.
      */
-    @AwaitsFix(bugUrl = "Keystore fails to save with resource busy")
-    public void test61DockerEnvironmentVariablePasswordFromFile() throws Exception {
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/76124")
+    public void test41DockerEnvironmentVariablePasswordFromFile() throws Exception {
         assumeTrue(distribution().isDocker());
 
         Path tempDir = null;
         try {
             tempDir = createTempDir(KeystoreManagementTests.class.getSimpleName());
 
-            String password = "keystore-password";
             String passwordFilename = "password.txt";
-            Files.writeString(tempDir.resolve(passwordFilename), password + "\n");
+            Files.writeString(tempDir.resolve(passwordFilename), KEYSTORE_PASSWORD + "\n");
             Files.setPosixFilePermissions(tempDir.resolve(passwordFilename), p600);
 
-            Path localConfigDir = getMountedLocalConfDirWithKeystore(password, installation.config);
+            Path localConfigDir = getMountedLocalConfDirWithKeystore(KEYSTORE_PASSWORD, installation.config);
 
             // restart ES with password and mounted config dir containing password protected keystore
             runContainer(
@@ -304,11 +267,11 @@ public class KeystoreManagementTests extends PackagingTestCase {
                 builder().volume(localConfigDir.resolve("config"), installation.config)
                     .volume(tempDir, "/run/secrets")
                     .envVar("KEYSTORE_PASSWORD_FILE", "/run/secrets/" + passwordFilename)
-                    .envVar("ELASTIC_PASSWORD", PASSWORD)
+                    .envVar("ELASTIC_PASSWORD", ELASTIC_PASSWORD)
             );
 
-            waitForElasticsearch(installation, USERNAME, PASSWORD);
-            ServerUtils.runElasticsearchTests(USERNAME, PASSWORD);
+            waitForElasticsearch(installation, "elastic", ELASTIC_PASSWORD);
+            runElasticsearchTestsAsElastic(ELASTIC_PASSWORD);
         } finally {
             if (tempDir != null) {
                 rm(tempDir);
@@ -320,12 +283,11 @@ public class KeystoreManagementTests extends PackagingTestCase {
      * Check that if we provide the wrong password for a mounted and password-protected
      * keystore, Elasticsearch doesn't start.
      */
-    @AwaitsFix(bugUrl = "Keystore fails to save with resource busy")
-    public void test62DockerEnvironmentVariableBadPassword() throws Exception {
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/76124")
+    public void test42DockerEnvironmentVariableBadPassword() throws Exception {
         assumeTrue(distribution().isDocker());
-        String password = "keystore-password";
 
-        Path localConfigPath = getMountedLocalConfDirWithKeystore(password, installation.config);
+        Path localConfigPath = getMountedLocalConfDirWithKeystore(KEYSTORE_PASSWORD, installation.config);
 
         // restart ES with password and mounted config dir containing password protected keystore
         Shell.Result r = runContainerExpectingFailure(
@@ -335,6 +297,38 @@ public class KeystoreManagementTests extends PackagingTestCase {
         assertThat(r.stderr, containsString(ERROR_INCORRECT_PASSWORD));
     }
 
+    public void test50CreateKeystoreManually() throws Exception {
+        // Run this test last so that removing the existing keystore doesn't make subsequent tests fail
+        rmKeystoreIfExists();
+        createKeystore(null);
+
+        final Installation.Executables bin = installation.executables();
+        verifyKeystorePermissions();
+
+        Shell.Result r = bin.keystoreTool.run("list");
+        assertThat(r.stdout, containsString("keystore.seed"));
+    }
+
+    public void test60AutoCreateKeystore() throws Exception {
+        // Run this test last so that removing the existing keystore doesn't make subsequent tests fail
+        assumeTrue("Packages and docker are installed with a keystore file", distribution.isArchive());
+        rmKeystoreIfExists();
+        // Elasticsearch was auto-configured for security. We need to remove that configuration as it depended on settings in the previous
+        // keystore
+        ServerUtils.disableSecurityFeatures(installation);
+
+        startElasticsearch();
+        stopElasticsearch();
+
+        Platforms.onWindows(() -> sh.chown(installation.config("elasticsearch.keystore")));
+
+        verifyKeystorePermissions();
+
+        final Installation.Executables bin = installation.executables();
+        Shell.Result r = bin.keystoreTool.run("list");
+        assertThat(r.stdout, containsString("keystore.seed"));
+    }
+
     /**
      * In the Docker context, it's a little bit tricky to get a password-protected
      * keystore. All of the utilities we'd want to use are on the Docker image.

+ 2 - 5
qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

@@ -8,7 +8,6 @@
 
 package org.elasticsearch.packaging.test;
 
-import org.apache.http.client.fluent.Request;
 import org.elasticsearch.packaging.util.FileUtils;
 import org.elasticsearch.packaging.util.Packages;
 import org.elasticsearch.packaging.util.Shell.Result;
@@ -40,8 +39,6 @@ import static org.elasticsearch.packaging.util.Packages.restartElasticsearch;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
 import static org.elasticsearch.packaging.util.Platforms.getOsRelease;
 import static org.elasticsearch.packaging.util.Platforms.isSystemd;
-import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
-import static org.elasticsearch.packaging.util.ServerUtils.runElasticsearchTests;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.Matchers.containsString;
@@ -126,7 +123,7 @@ public class PackageTests extends PackagingTestCase {
 
         startElasticsearch();
 
-        final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+        final String nodesResponse = makeRequest("http://localhost:9200/_nodes");
         assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
 
         stopElasticsearch();
@@ -288,7 +285,7 @@ public class PackageTests extends PackagingTestCase {
 
             startElasticsearch();
 
-            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            final String nodesResponse = makeRequest("http://localhost:9200/_nodes");
             assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
             assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
 

+ 24 - 9
qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java

@@ -12,13 +12,13 @@ import org.apache.http.client.fluent.Request;
 import org.apache.http.entity.ContentType;
 import org.elasticsearch.packaging.util.Distribution;
 import org.elasticsearch.packaging.util.Packages;
+import org.elasticsearch.packaging.util.ServerUtils;
 
 import java.nio.file.Paths;
 
 import static org.elasticsearch.packaging.util.Packages.assertInstalled;
 import static org.elasticsearch.packaging.util.Packages.installPackage;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
-import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
 import static org.hamcrest.Matchers.containsString;
 
 public class PackageUpgradeTests extends PackagingTestCase {
@@ -44,25 +44,25 @@ public class PackageUpgradeTests extends PackagingTestCase {
         startElasticsearch();
 
         // create indexes explicitly with 0 replicas so when restarting we can reach green state
-        makeRequest(
+        ServerUtils.makeRequest(
             Request.Put("http://localhost:9200/library")
                 .bodyString("{\"settings\":{\"index\":{\"number_of_replicas\":0}}}", ContentType.APPLICATION_JSON)
         );
-        makeRequest(
+        ServerUtils.makeRequest(
             Request.Put("http://localhost:9200/library2")
                 .bodyString("{\"settings\":{\"index\":{\"number_of_replicas\":0}}}", ContentType.APPLICATION_JSON)
         );
 
         // add some docs
-        makeRequest(
+        ServerUtils.makeRequest(
             Request.Post("http://localhost:9200/library/_doc/1?refresh=true&pretty")
                 .bodyString("{ \"title\": \"Elasticsearch - The Definitive Guide\"}", ContentType.APPLICATION_JSON)
         );
-        makeRequest(
+        ServerUtils.makeRequest(
             Request.Post("http://localhost:9200/library/_doc/2?refresh=true&pretty")
                 .bodyString("{ \"title\": \"Brave New World\"}", ContentType.APPLICATION_JSON)
         );
-        makeRequest(
+        ServerUtils.makeRequest(
             Request.Post("http://localhost:9200/library2/_doc/1?refresh=true&pretty")
                 .bodyString("{ \"title\": \"The Left Hand of Darkness\"}", ContentType.APPLICATION_JSON)
         );
@@ -91,11 +91,26 @@ public class PackageUpgradeTests extends PackagingTestCase {
     private void assertDocsExist() throws Exception {
         // We can properly handle this as part of https://github.com/elastic/elasticsearch/issues/75940
         // For now we can use elastic with "keystore.seed" as we set it explicitly in PackageUpgradeTests#test11ModifyKeystore
-        String response1 = makeRequest(Request.Get("http://localhost:9200/library/_doc/1?pretty"), "elastic", "keystore_seed", null);
+        String response1 = ServerUtils.makeRequest(
+            Request.Get("http://localhost:9200/library/_doc/1?pretty"),
+            "elastic",
+            "keystore_seed",
+            null
+        );
         assertThat(response1, containsString("Elasticsearch"));
-        String response2 = makeRequest(Request.Get("http://localhost:9200/library/_doc/2?pretty"), "elastic", "keystore_seed", null);
+        String response2 = ServerUtils.makeRequest(
+            Request.Get("http://localhost:9200/library/_doc/2?pretty"),
+            "elastic",
+            "keystore_seed",
+            null
+        );
         assertThat(response2, containsString("World"));
-        String response3 = makeRequest(Request.Get("http://localhost:9200/library2/_doc/1?pretty"), "elastic", "keystore_seed", null);
+        String response3 = ServerUtils.makeRequest(
+            Request.Get("http://localhost:9200/library2/_doc/1?pretty"),
+            "elastic",
+            "keystore_seed",
+            null
+        );
         assertThat(response3, containsString("Darkness"));
     }
 }

+ 197 - 4
qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -15,23 +15,29 @@ import com.carrotsearch.randomizedtesting.annotations.TestGroup;
 import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
 import com.carrotsearch.randomizedtesting.annotations.Timeout;
 
+import org.apache.http.client.fluent.Request;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.packaging.util.Archives;
 import org.elasticsearch.packaging.util.Distribution;
+import org.elasticsearch.packaging.util.FileMatcher;
 import org.elasticsearch.packaging.util.FileUtils;
 import org.elasticsearch.packaging.util.Installation;
 import org.elasticsearch.packaging.util.Packages;
 import org.elasticsearch.packaging.util.Platforms;
+import org.elasticsearch.packaging.util.ServerUtils;
 import org.elasticsearch.packaging.util.Shell;
 import org.elasticsearch.packaging.util.docker.Docker;
+import org.elasticsearch.packaging.util.docker.DockerFileMatcher;
 import org.elasticsearch.packaging.util.docker.DockerShell;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -56,19 +62,35 @@ import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
 import static org.elasticsearch.packaging.util.Cleanup.cleanEverything;
 import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileExists;
+import static org.elasticsearch.packaging.util.FileMatcher.Fileness.Directory;
+import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File;
+import static org.elasticsearch.packaging.util.FileMatcher.p660;
+import static org.elasticsearch.packaging.util.FileMatcher.p750;
 import static org.elasticsearch.packaging.util.FileUtils.append;
+import static org.elasticsearch.packaging.util.FileUtils.rm;
+import static org.elasticsearch.packaging.util.docker.Docker.copyFromContainer;
 import static org.elasticsearch.packaging.util.docker.Docker.ensureImageIsLoaded;
 import static org.elasticsearch.packaging.util.docker.Docker.removeContainer;
 import static org.hamcrest.CoreMatchers.anyOf;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.junit.Assume.assumeFalse;
 import static org.junit.Assume.assumeTrue;
 
@@ -115,6 +137,7 @@ public abstract class PackagingTestCase extends Assert {
 
     // the current installation of the distribution being tested
     protected static Installation installation;
+    protected static Tuple<String, String> fileSuperuserForInstallation;
 
     private static boolean failed;
 
@@ -239,6 +262,7 @@ public abstract class PackagingTestCase extends Assert {
 
     protected static void cleanup() throws Exception {
         installation = null;
+        fileSuperuserForInstallation = null;
         cleanEverything();
     }
 
@@ -387,7 +411,19 @@ public abstract class PackagingTestCase extends Assert {
      * @throws Exception if Elasticsearch can't start
      */
     public void startElasticsearch() throws Exception {
-        awaitElasticsearchStartup(runElasticsearchStartCommand(null, true, false));
+        try {
+            awaitElasticsearchStartup(runElasticsearchStartCommand(null, true, false));
+        } catch (Exception e) {
+            if (Files.exists(installation.home.resolve("elasticsearch.pid"))) {
+                String pid = FileUtils.slurp(installation.home.resolve("elasticsearch.pid")).trim();
+                logger.info("elasticsearch process ({}) failed to start", pid);
+                if (sh.run("jps").stdout.contains(pid)) {
+                    logger.info("Dumping jstack of elasticsearch process ({}) ", pid);
+                    sh.runIgnoreExitCode("jstack " + pid);
+                }
+            }
+            throw e;
+        }
     }
 
     public void assertElasticsearchFailure(Shell.Result result, String expectedMessage, Packages.JournaldWrapper journaldWrapper) {
@@ -421,8 +457,8 @@ public abstract class PackagingTestCase extends Assert {
             sh.runIgnoreExitCode("Wait-Process -Timeout " + Archives.ES_STARTUP_SLEEP_TIME_SECONDS + " -Id " + wrapperPid);
             sh.runIgnoreExitCode(
                 "Get-EventSubscriber | "
-                    + "where {($_.EventName -eq 'OutputDataReceived' -Or $_.EventName -eq 'ErrorDataReceived' |"
-                    + "Unregister-EventSubscriber -Force"
+                    + "Where-Object {($_.EventName -eq 'OutputDataReceived') -or ($_.EventName -eq 'ErrorDataReceived')} | "
+                    + "Unregister-Event -Force"
             );
             assertThat(FileUtils.slurp(Archives.getPowershellErrorPath(installation)), anyOf(stringMatchers));
 
@@ -433,6 +469,50 @@ public abstract class PackagingTestCase extends Assert {
         }
     }
 
+    public void setFileSuperuser(String username, String password) {
+        assertThat(installation, Matchers.not(Matchers.nullValue()));
+        assertThat(fileSuperuserForInstallation, Matchers.nullValue());
+        Shell.Result result = sh.run(
+            installation.executables().usersTool + " useradd " + username + " -p " + password + " -r " + "superuser"
+        );
+        assertThat(result.isSuccess(), is(true));
+        fileSuperuserForInstallation = new Tuple<>(username, password);
+    }
+
+    public void runElasticsearchTestsAsElastic(String elasticPassword) throws Exception {
+        ServerUtils.runElasticsearchTests("elastic", elasticPassword, ServerUtils.getCaCert(installation));
+    }
+
+    public void runElasticsearchTests() throws Exception {
+        ServerUtils.runElasticsearchTests(
+            fileSuperuserForInstallation != null ? fileSuperuserForInstallation.v1() : null,
+            fileSuperuserForInstallation != null ? fileSuperuserForInstallation.v2() : null,
+            ServerUtils.getCaCert(installation)
+        );
+    }
+
+    public String makeRequest(String request) throws Exception {
+        return ServerUtils.makeRequest(
+            Request.Get(request),
+            fileSuperuserForInstallation != null ? fileSuperuserForInstallation.v1() : null,
+            fileSuperuserForInstallation != null ? fileSuperuserForInstallation.v2() : null,
+            ServerUtils.getCaCert(installation)
+        );
+    }
+
+    public String makeRequestAsElastic(String request, String elasticPassword) throws Exception {
+        return ServerUtils.makeRequest(Request.Get(request), "elastic", elasticPassword, ServerUtils.getCaCert(installation));
+    }
+
+    public int makeRequestAsElastic(String elasticPassword) throws Exception {
+        return ServerUtils.makeRequestAndGetStatus(
+            Request.Get("https://localhost:9200"),
+            "elastic",
+            elasticPassword,
+            ServerUtils.getCaCert(installation)
+        );
+    }
+
     public static Path getRootTempDir() {
         if (distribution().isPackage()) {
             // The custom config directory is not under /tmp or /var/tmp because
@@ -470,7 +550,8 @@ public abstract class PackagingTestCase extends Assert {
         Path tempConf = tempDir.resolve("elasticsearch");
         FileUtils.copyDirectory(installation.config, tempConf);
 
-        Platforms.onLinux(() -> sh.run("chown -R elasticsearch:elasticsearch " + tempDir));
+        // this is what install does
+        sh.chown(tempDir);
 
         if (distribution.isPackage()) {
             Files.copy(installation.envFile, tempDir.resolve("elasticsearch.bk"), StandardCopyOption.COPY_ATTRIBUTES);// backup
@@ -479,6 +560,19 @@ public abstract class PackagingTestCase extends Assert {
             sh.getEnv().put("ES_PATH_CONF", tempConf.toString());
         }
 
+        // Auto-configuration file paths are absolute so we need to replace them in the config now that we copied them to tempConf
+        // if auto-configuration has happened. Otherwise, the action below is a no-op.
+        Path yml = tempConf.resolve("elasticsearch.yml");
+        List<String> lines;
+        try (Stream<String> allLines = Files.readAllLines(yml).stream()) {
+            lines = allLines.map(l -> {
+                if (l.contains(installation.config.toString())) {
+                    return l.replace(installation.config.toString(), tempConf.toString());
+                }
+                return l;
+            }).collect(Collectors.toList());
+        }
+        Files.write(yml, lines, TRUNCATE_EXISTING);
         action.accept(tempConf);
         if (distribution.isPackage()) {
             IOUtils.rm(installation.envFile);
@@ -489,6 +583,17 @@ public abstract class PackagingTestCase extends Assert {
         IOUtils.rm(tempDir);
     }
 
+    public void withCustomConfigOwner(String tempOwner, Predicate<Distribution.Platform> predicate, CheckedRunnable<Exception> action)
+        throws Exception {
+        if (predicate.test(installation.distribution.platform)) {
+            sh.chown(installation.config, tempOwner);
+            action.run();
+            sh.chown(installation.config);
+        } else {
+            action.run();
+        }
+    }
+
     /**
      * Manually set the heap size with a jvm.options.d file. This will be reset before each test.
      */
@@ -549,4 +654,92 @@ public abstract class PackagingTestCase extends Assert {
             throw e;
         }
     }
+
+    /**
+     * Validates that the installation {@code es} has been auto-configured. This applies to archives and docker only,
+     * packages have nuances that justify their own version.
+     * @param es the {@link Installation} to check
+     */
+    public void verifySecurityAutoConfigured(Installation es) throws Exception {
+        Optional<String> autoConfigDirName = getAutoConfigDirName(es);
+        assertThat(autoConfigDirName.isPresent(), Matchers.is(true));
+        final List<String> configLines;
+        if (es.distribution.isArchive()) {
+            // We chown the installation on Windows to Administrators so that we can auto-configure it.
+            String owner = Platforms.WINDOWS ? "BUILTIN\\Administrators" : "elasticsearch";
+            assertThat(es.config(autoConfigDirName.get()), FileMatcher.file(Directory, owner, owner, p750));
+            Stream.of("http_keystore_local_node.p12", "http_ca.crt", "transport_keystore_all_nodes.p12")
+                .forEach(file -> assertThat(es.config(autoConfigDirName.get()).resolve(file), FileMatcher.file(File, owner, owner, p660)));
+            configLines = Files.readAllLines(es.config("elasticsearch.yml"));
+        } else {
+            assertThat(es.config(autoConfigDirName.get()), DockerFileMatcher.file(Directory, "elasticsearch", "root", p750));
+            Stream.of("http_keystore_local_node.p12", "http_ca.crt", "transport_keystore_all_nodes.p12")
+                .forEach(
+                    file -> assertThat(
+                        es.config(autoConfigDirName.get()).resolve(file),
+                        DockerFileMatcher.file(File, "elasticsearch", "root", p660)
+                    )
+                );
+            Path localTempDir = createTempDir("docker-config");
+            copyFromContainer(es.config("elasticsearch.yml"), localTempDir.resolve("docker_elasticsearch.yml"));
+            configLines = Files.readAllLines(localTempDir.resolve("docker_elasticsearch.yml"));
+            rm(localTempDir.resolve("docker_elasticsearch.yml"));
+            rm(localTempDir);
+        }
+
+        assertThat(configLines, hasItem("xpack.security.enabled: true"));
+        assertThat(configLines, hasItem("xpack.security.http.ssl.enabled: true"));
+        assertThat(configLines, hasItem("xpack.security.transport.ssl.enabled: true"));
+
+        assertThat(configLines, hasItem("xpack.security.enrollment.enabled: true"));
+        assertThat(configLines, hasItem("xpack.security.transport.ssl.verification_mode: certificate"));
+        assertThat(
+            configLines,
+            hasItem(
+                "xpack.security.transport.ssl.keystore.path: "
+                    + es.config(autoConfigDirName.get()).resolve("transport_keystore_all_nodes.p12")
+            )
+        );
+        assertThat(
+            configLines,
+            hasItem(
+                "xpack.security.transport.ssl.truststore.path: "
+                    + es.config(autoConfigDirName.get()).resolve("transport_keystore_all_nodes.p12")
+            )
+        );
+        assertThat(
+            configLines,
+            hasItem("xpack.security.http.ssl.keystore.path: " + es.config(autoConfigDirName.get()).resolve("http_keystore_local_node.p12"))
+        );
+        if (es.distribution.isDocker() == false) {
+            assertThat(configLines, hasItem("http.host: [_local_, _site_]"));
+        }
+    }
+
+    /**
+     * Validates that the installation {@code es} has not been auto-configured. This applies to archives and docker only,
+     * packages have nuances that justify their own version.
+     * @param es the {@link Installation} to check
+     */
+    public static void verifySecurityNotAutoConfigured(Installation es) throws Exception {
+        assertThat(getAutoConfigDirName(es).isPresent(), Matchers.is(false));
+        List<String> configLines = Files.readAllLines(es.config("elasticsearch.yml"));
+        assertThat(configLines, not(contains(containsString("automatically generated in order to configure Security"))));
+        Path caCert = ServerUtils.getCaCert(installation);
+        if (caCert != null) {
+            assertThat(caCert.toString(), Matchers.not(Matchers.containsString("tls_auto_config_initial_node")));
+        }
+    }
+
+    public static Optional<String> getAutoConfigDirName(Installation es) {
+        final Shell.Result lsResult;
+        if (es.distribution.platform.equals(Distribution.Platform.WINDOWS)) {
+            lsResult = sh.run("Get-ChildItem -Path " + es.config + " -Name");
+        } else {
+            lsResult = sh.run("find \"" + es.config + "\" -type d -maxdepth 1");
+        }
+        assertNotNull(lsResult.stdout);
+        return Arrays.stream(lsResult.stdout.split("\n")).filter(f -> f.contains("tls_auto_config_initial_node_")).findFirst();
+    }
+
 }

+ 6 - 4
qa/os/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java

@@ -8,7 +8,6 @@
 
 package org.elasticsearch.packaging.test;
 
-import org.apache.http.client.fluent.Request;
 import org.elasticsearch.packaging.test.PackagingTestCase.AwaitsFix;
 import org.elasticsearch.packaging.util.Installation;
 import org.elasticsearch.packaging.util.Platforms;
@@ -19,7 +18,6 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
-import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assume.assumeFalse;
@@ -58,6 +56,7 @@ public class PluginCliTests extends PackagingTestCase {
 
     public void test10Install() throws Exception {
         install();
+        setFileSuperuser("test_superuser", "test_superuser_password");
     }
 
     public void test20SymlinkPluginsDir() throws Exception {
@@ -69,13 +68,16 @@ public class PluginCliTests extends PackagingTestCase {
         Path linkedPlugins = createTempDir("symlinked-plugins");
         Platforms.onLinux(() -> sh.run("chown elasticsearch:elasticsearch " + linkedPlugins.toString()));
         Files.createSymbolicLink(pluginsDir, linkedPlugins);
+        // Packaged installation don't get autoconfigured yet
+        // TODO: Remove this in https://github.com/elastic/elasticsearch/pull/75144
+        String protocol = distribution.isPackage() ? "http" : "https";
         assertWithExamplePlugin(installResult -> {
             assertWhileRunning(() -> {
-                final String pluginsResponse = makeRequest(Request.Get("http://localhost:9200/_cat/plugins?h=component")).strip();
+                final String pluginsResponse = makeRequest(protocol + "://localhost:9200/_cat/plugins?h=component").strip();
                 assertThat(pluginsResponse, equalTo(EXAMPLE_PLUGIN_NAME));
 
                 String settingsPath = "_cluster/settings?include_defaults&filter_path=defaults.custom.simple";
-                final String settingsResponse = makeRequest(Request.Get("http://localhost:9200/" + settingsPath)).strip();
+                final String settingsResponse = makeRequest(protocol + "://localhost:9200/" + settingsPath).strip();
                 assertThat(settingsResponse, equalTo("{\"defaults\":{\"custom\":{\"simple\":\"foo\"}}}"));
             });
         });

+ 2 - 1
qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java

@@ -93,6 +93,7 @@ public class WindowsServiceTests extends PackagingTestCase {
     public void test10InstallArchive() throws Exception {
         installation = installArchive(sh, distribution());
         verifyArchiveInstallation(installation, distribution());
+        setFileSuperuser("test_superuser", "test_superuser_password");
         serviceScript = installation.bin("elasticsearch-service.bat").toString();
     }
 
@@ -172,7 +173,7 @@ public class WindowsServiceTests extends PackagingTestCase {
     // NOTE: service description is not attainable through any powershell api, so checking it is not possible...
     public void assertStartedAndStop() throws Exception {
         ServerUtils.waitForElasticsearch(installation);
-        ServerUtils.runElasticsearchTests();
+        runElasticsearchTests();
 
         assertCommand(serviceScript + " stop");
         assertService(DEFAULT_ID, "Stopped", DEFAULT_DISPLAY_NAME);

+ 7 - 8
qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java

@@ -54,7 +54,7 @@ public class Archives {
 
     /** This is an arbitrarily chosen value that gives Elasticsearch time to log Bootstrap
      *  errors to the console if they occur before the logging framework is initialized. */
-    public static final String ES_STARTUP_SLEEP_TIME_SECONDS = "10";
+    public static final String ES_STARTUP_SLEEP_TIME_SECONDS = "15";
 
     public static Installation installArchive(Shell sh, Distribution distribution) throws Exception {
         return installArchive(sh, distribution, getDefaultArchiveInstallPath(), getCurrentVersion());
@@ -107,9 +107,6 @@ public class Archives {
 
         Installation installation = Installation.ofArchive(sh, distribution, fullInstallPath);
         ServerUtils.disableGeoIpDownloader(installation);
-        // TODO: Adjust all tests so that they can run with security on, which is the default behavior
-        // https://github.com/elastic/elasticsearch/issues/75940
-        ServerUtils.possiblyDisableSecurityFeatures(installation);
 
         return installation;
     }
@@ -200,7 +197,6 @@ public class Archives {
             "elasticsearch-certutil",
             "elasticsearch-croneval",
             "elasticsearch-saml-metadata",
-            "elasticsearch-security-config",
             "elasticsearch-setup-passwords",
             "elasticsearch-sql-cli",
             "elasticsearch-syskeygen",
@@ -241,6 +237,7 @@ public class Archives {
         if (daemonize) {
             command.add("-d");
         }
+        command.add("-v"); // verbose auto-configuration
         String script = String.format(
             Locale.ROOT,
             "expect -c \"$(cat<<EXPECT\n"
@@ -292,6 +289,7 @@ public class Archives {
             if (daemonize) {
                 command.add("-d");
             }
+            command.add("-v"); // verbose auto-configuration
             command.add("-p");
             command.add(pidFile.toString());
             if (keystorePassword != null) {
@@ -321,7 +319,7 @@ public class Archives {
                     + "$processInfo.FileName = '"
                     + bin.elasticsearch
                     + "'; "
-                    + "$processInfo.Arguments = '-p "
+                    + "$processInfo.Arguments = '-v -p "
                     + installation.home.resolve("elasticsearch.pid")
                     + "'; "
                     + powerShellProcessUserSetup
@@ -367,6 +365,7 @@ public class Archives {
                 command.add("echo '" + keystorePassword + "' |");
             }
             command.add(bin.elasticsearch.toString());
+            command.add("-v"); // verbose auto-configuration
             command.add("-p");
             command.add(installation.home.resolve("elasticsearch.pid").toString());
             return sh.runIgnoreExitCode(String.join(" ", command));
@@ -396,8 +395,8 @@ public class Archives {
             // Clear the asynchronous event handlers
             sh.runIgnoreExitCode(
                 "Get-EventSubscriber | "
-                    + "where {($_.EventName -eq 'OutputDataReceived' -Or $_.EventName -eq 'ErrorDataReceived' |"
-                    + "Unregister-EventSubscriber -Force"
+                    + "Where-Object {($_.EventName -eq 'OutputDataReceived') -or ($_.EventName -eq 'ErrorDataReceived')} | "
+                    + "Unregister-Event -Force"
             );
         });
         if (Files.exists(pidFile)) {

+ 7 - 2
qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java

@@ -8,6 +8,9 @@
 
 package org.elasticsearch.packaging.util;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -23,6 +26,8 @@ import static org.elasticsearch.packaging.util.Platforms.isRPM;
 
 public class Cleanup {
 
+    protected static final Logger logger = LogManager.getLogger(Cleanup.class);
+
     private static final List<String> ELASTICSEARCH_FILES_LINUX = Arrays.asList(
         "/usr/share/elasticsearch",
         "/etc/elasticsearch/elasticsearch.keystore",
@@ -67,9 +72,9 @@ public class Cleanup {
             sh.runIgnoreExitCode("groupdel elasticsearch");
         });
         // when we run es as a role user on windows, add the equivalent here
-
         // delete files that may still exist
-        lsGlob(getRootTempDir(), "elasticsearch*").forEach(FileUtils::rm);
+
+        lsGlob(getRootTempDir(), "elasticsearch*").forEach(Platforms.WINDOWS ? FileUtils::rmWithRetries : FileUtils::rm);
         final List<String> filesToDelete = Platforms.WINDOWS ? ELASTICSEARCH_FILES_WINDOWS : ELASTICSEARCH_FILES_LINUX;
         // windows needs leniency due to asinine releasing of file locking async from a process exiting
         Consumer<? super Path> rm = Platforms.WINDOWS ? FileUtils::rmWithRetries : FileUtils::rm;

+ 0 - 1
qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java

@@ -185,7 +185,6 @@ public class Installation {
         public final Executable cronevalTool = new Executable("elasticsearch-croneval");
         public final Executable shardTool = new Executable("elasticsearch-shard");
         public final Executable nodeTool = new Executable("elasticsearch-node");
-        public final Executable securityConfigTool = new Executable("elasticsearch-security-config");
         public final Executable setupPasswordsTool = new Executable("elasticsearch-setup-passwords");
         public final Executable resetElasticPasswordTool = new Executable("elasticsearch-reset-elastic-password");
         public final Executable sqlCli = new Executable("elasticsearch-sql-cli");

+ 1 - 1
qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java

@@ -96,7 +96,7 @@ public class Packages {
         }
         // https://github.com/elastic/elasticsearch/issues/75940
         // TODO Figure out how to run all packaging tests with security enabled which is now the default behavior
-        ServerUtils.possiblyDisableSecurityFeatures(installation);
+        ServerUtils.disableSecurityFeatures(installation);
         return installation;
     }
 

+ 124 - 93
qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -24,6 +24,7 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.util.EntityUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.packaging.test.PackagingTestCase;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -35,7 +36,6 @@ import java.nio.file.Path;
 import java.security.KeyStore;
 import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -47,12 +47,13 @@ import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManagerFactory;
 
-import static java.nio.file.StandardOpenOption.APPEND;
-import static java.nio.file.StandardOpenOption.CREATE;
 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
+import static org.elasticsearch.packaging.util.docker.Docker.copyFromContainer;
 import static org.elasticsearch.packaging.util.docker.Docker.dockerShell;
+import static org.elasticsearch.packaging.util.docker.Docker.findInContainer;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
 
 public class ServerUtils {
 
@@ -64,9 +65,10 @@ public class ServerUtils {
     private static final long waitTime = TimeUnit.MINUTES.toMillis(3);
     private static final long timeoutLength = TimeUnit.SECONDS.toMillis(30);
     private static final long requestInterval = TimeUnit.SECONDS.toMillis(5);
+    private static final long dockerWaitForSecurityIndex = TimeUnit.SECONDS.toMillis(25);
 
     public static void waitForElasticsearch(Installation installation) throws Exception {
-        boolean securityEnabled;
+        final boolean securityEnabled;
 
         if (installation.distribution.isDocker() == false) {
             Path configFilePath = installation.config("elasticsearch.yml");
@@ -85,11 +87,13 @@ public class ServerUtils {
         }
 
         if (securityEnabled) {
+            logger.info("Waiting for elasticsearch WITH Security enabled");
             // with security enabled, we may or may not have setup a user/pass, so we use a more generic port being available check.
             // this isn't as good as a health check, but long term all this waiting should go away when node startup does not
             // make the http port available until the system is really ready to serve requests
-            waitForXpack();
+            waitForXpack(installation);
         } else {
+            logger.info("Waiting for elasticsearch WITHOUT Security enabled");
             waitForElasticsearch("green", null, installation, null, null, null);
         }
     }
@@ -141,8 +145,8 @@ public class ServerUtils {
         return executor.execute(request).returnResponse();
     }
 
-    // polls every second for Elasticsearch to be running on 9200
-    private static void waitForXpack() {
+    // polls every two seconds for Elasticsearch to be running on 9200
+    private static void waitForXpack(Installation installation) {
         int retries = 60;
         while (retries > 0) {
             retries -= 1;
@@ -152,19 +156,39 @@ public class ServerUtils {
                 // ignore, only want to establish a connection
             }
             try {
-                Thread.sleep(1000);
+                Thread.sleep(2000);
             } catch (InterruptedException interrupted) {
                 Thread.currentThread().interrupt();
                 return;
             }
         }
+        if (installation != null) {
+            FileUtils.logAllLogs(installation.logs, logger);
+        }
+
         throw new RuntimeException("Elasticsearch (with x-pack) did not start");
     }
 
+    public static Path getCaCert(Installation installation) throws IOException {
+        if (installation.distribution.isDocker()) {
+            final Path tempDir = PackagingTestCase.createTempDir("docker-ssl");
+            final Path autoConfigurationDir = findInContainer(installation.config, "d", "\"tls_auto_config_initial_node_*\"");
+            if (autoConfigurationDir != null) {
+                final Path hostHttpCaCert = tempDir.resolve("http_ca.crt");
+                copyFromContainer(autoConfigurationDir.resolve("http_ca.crt"), hostHttpCaCert);
+                return hostHttpCaCert;
+            } else {
+                return null;
+            }
+        } else {
+            return getCaCert(installation.config);
+        }
+    }
+
     public static Path getCaCert(Path configPath) throws IOException {
         boolean enrollmentEnabled = false;
         boolean httpSslEnabled = false;
-        Path caCert = configPath.resolve("certs/ca/ca.crt");
+        Path caCert = configPath.resolve("certs").resolve("ca").resolve("ca.crt");
         Path configFilePath = configPath.resolve("elasticsearch.yml");
         if (Files.exists(configFilePath)) {
             // In docker we might not even have a file, and if we do it's not in the host's FS
@@ -174,11 +198,10 @@ public class ServerUtils {
         }
         if (enrollmentEnabled && httpSslEnabled) {
             assert Files.exists(caCert) == false;
-            Path autoConfigTlsDir = Files.list(configPath)
-                .filter(p -> p.getFileName().toString().startsWith("tls_auto_config_initial_node_"))
-                .findFirst()
-                .get();
-            caCert = autoConfigTlsDir.resolve("http_ca.crt");
+            List<Path> allAutoconfTLS = FileUtils.lsGlob(configPath, "tls_auto_config_initial_node_*");
+            assertThat(allAutoconfTLS.size(), is(1));
+            Path autoconfTLSDir = allAutoconfTLS.get(0);
+            caCert = autoconfTLSDir.resolve("http_ca.crt");
             logger.info("Node has TLS auto-configured [" + caCert + "]");
             assert Files.exists(caCert);
         } else if (Files.exists(caCert) == false) {
@@ -197,7 +220,7 @@ public class ServerUtils {
         Path caCert
     ) throws Exception {
         Objects.requireNonNull(status);
-
+        boolean shouldRetryOnAuthNFailure = false;
         // we loop here rather than letting httpclient handle retries so we can measure the entire waiting time
         final long startTime = System.currentTimeMillis();
         long lastRequest = 0;
@@ -205,7 +228,7 @@ public class ServerUtils {
         boolean started = false;
         Throwable thrownException = null;
         if (caCert == null) {
-            caCert = getCaCert(installation.config);
+            caCert = getCaCert(installation);
         }
 
         while (started == false && timeElapsed < waitTime) {
@@ -220,13 +243,28 @@ public class ServerUtils {
                         password,
                         caCert
                     );
-
                     if (response.getStatusLine().getStatusCode() >= 300) {
-                        final String statusLine = response.getStatusLine().toString();
-                        final String body = EntityUtils.toString(response.getEntity());
-                        throw new RuntimeException("Connecting to elasticsearch cluster health API failed:\n" + statusLine + "\n" + body);
+                        // We create the security index on startup (in order to create an enrollment token and/or set the elastic password)
+                        // In Docker, even when the ELASTIC_PASSWORD is set, when the security index exists and we get an authN attempt as
+                        // `elastic` , the reserved realm checks the security index first. It can happen that we check the security index
+                        // too early after the security index creation in DockerTests causing an UnavailableShardsException. We retry
+                        // authentication errors for a couple of seconds just to verify this is not the case.
+                        if (installation.distribution.isDocker()
+                            && timeElapsed < dockerWaitForSecurityIndex
+                            && response.getStatusLine().getStatusCode() == 401) {
+                            logger.info(
+                                "Authentication against docker failed (possibly due to UnavailableShardsException for the security index)"
+                                    + ", retrying..."
+                            );
+                            shouldRetryOnAuthNFailure = true;
+                        } else {
+                            final String statusLine = response.getStatusLine().toString();
+                            final String body = EntityUtils.toString(response.getEntity());
+                            throw new RuntimeException(
+                                "Connecting to elasticsearch cluster health API failed:\n" + statusLine + "\n" + body
+                            );
+                        }
                     }
-
                     started = true;
 
                 } catch (IOException e) {
@@ -251,64 +289,59 @@ public class ServerUtils {
             throw new RuntimeException("Elasticsearch did not start", thrownException);
         }
 
-        final String url;
-        if (index == null) {
-            url = (caCert != null ? "https" : "http")
-                + "://localhost:9200/_cluster/health?wait_for_status="
-                + status
-                + "&timeout=60s"
-                + "&pretty";
-        } else {
-            url = (caCert != null ? "https" : "http")
-                + "://localhost:9200/_cluster/health/"
-                + index
-                + "?wait_for_status="
-                + status
-                + "&timeout=60s&pretty";
-        }
+        if (shouldRetryOnAuthNFailure == false) {
+            final String url;
+            if (index == null) {
+                url = (caCert != null ? "https" : "http")
+                    + "://localhost:9200/_cluster/health?wait_for_status="
+                    + status
+                    + "&timeout=60s"
+                    + "&pretty";
+            } else {
+                url = (caCert != null ? "https" : "http")
+                    + "://localhost:9200/_cluster/health/"
+                    + index
+                    + "?wait_for_status="
+                    + status
+                    + "&timeout=60s&pretty";
+            }
 
-        final String body = makeRequest(Request.Get(url), username, password, caCert);
-        assertThat("cluster health response must contain desired status", body, containsString(status));
+            final String body = makeRequest(Request.Get(url), username, password, caCert);
+            assertThat("cluster health response must contain desired status", body, containsString(status));
+        }
     }
 
     public static void runElasticsearchTests() throws Exception {
-        makeRequest(
-            Request.Post("http://localhost:9200/library/_doc/1?refresh=true&pretty")
-                .bodyString("{ \"title\": \"Book #1\", \"pages\": 123 }", ContentType.APPLICATION_JSON)
-        );
-
-        makeRequest(
-            Request.Post("http://localhost:9200/library/_doc/2?refresh=true&pretty")
-                .bodyString("{ \"title\": \"Book #2\", \"pages\": 456 }", ContentType.APPLICATION_JSON)
-        );
-
-        String count = makeRequest(Request.Get("http://localhost:9200/_count?pretty"));
-        assertThat(count, containsString("\"count\" : 2"));
-
-        makeRequest(Request.Delete("http://localhost:9200/library"));
+        runElasticsearchTests(null, null, null);
     }
 
-    public static void runElasticsearchTests(String username, String password) throws Exception {
+    public static void runElasticsearchTests(String username, String password, Path caCert) throws Exception {
+
         makeRequest(
-            Request.Post("http://localhost:9200/library/_doc/1?refresh=true&pretty")
+            Request.Post((caCert != null ? "https" : "http") + "://localhost:9200/library/_doc/1?refresh=true&pretty")
                 .bodyString("{ \"title\": \"Book #1\", \"pages\": 123 }", ContentType.APPLICATION_JSON),
             username,
             password,
-            null
+            caCert
         );
 
         makeRequest(
-            Request.Post("http://localhost:9200/library/_doc/2?refresh=true&pretty")
+            Request.Post((caCert != null ? "https" : "http") + "://localhost:9200/library/_doc/2?refresh=true&pretty")
                 .bodyString("{ \"title\": \"Book #2\", \"pages\": 456 }", ContentType.APPLICATION_JSON),
             username,
             password,
-            null
+            caCert
         );
 
-        String count = makeRequest(Request.Get("http://localhost:9200/_count?pretty"), username, password, null);
+        String count = makeRequest(
+            Request.Get((caCert != null ? "https" : "http") + "://localhost:9200/library/_count?pretty"),
+            username,
+            password,
+            caCert
+        );
         assertThat(count, containsString("\"count\" : 2"));
 
-        makeRequest(Request.Delete("http://localhost:9200/library"), username, password, null);
+        makeRequest(Request.Delete((caCert != null ? "https" : "http") + "://localhost:9200/library"), username, password, caCert);
     }
 
     public static String makeRequest(Request request) throws Exception {
@@ -332,66 +365,64 @@ public class ServerUtils {
     }
 
     public static void disableGeoIpDownloader(Installation installation) throws IOException {
-        List<String> yaml = Collections.singletonList("ingest.geoip.downloader.enabled: false");
-        Path yml = installation.config("elasticsearch.yml");
-        try (Stream<String> lines = Files.readAllLines(yml).stream()) {
-            if (lines.noneMatch(s -> s.startsWith("ingest.geoip.downloader.enabled"))) {
-                Files.write(yml, yaml, CREATE, APPEND);
-            }
-        }
+        addSettingToExistingConfiguration(installation, "ingest.geoip.downloader.enabled", "false");
     }
 
     public static void enableGeoIpDownloader(Installation installation) throws IOException {
-        Path yml = installation.config("elasticsearch.yml");
-        List<String> lines;
-        try (Stream<String> allLines = Files.readAllLines(yml).stream()) {
-            lines = allLines.filter(s -> s.startsWith("ingest.geoip.downloader.enabled") == false).collect(Collectors.toList());
-        }
-        Files.write(yml, lines, TRUNCATE_EXISTING);
+        removeSettingFromExistingConfiguration(installation, "ingest.geoip.downloader.enabled");
     }
 
     /**
-     * Explicitly disables security if the existing configuration didn't already have an explicit value for the
-     * xpack.security.enabled setting
+     * Explicitly disables security features
      */
-    public static void possiblyDisableSecurityFeatures(Installation installation) throws IOException {
-        List<String> configLines = Collections.singletonList("xpack.security.enabled: false");
+    public static void disableSecurityFeatures(Installation installation) throws IOException {
+        List<String> disabledSecurityFeatures = List.of(
+            "xpack.security.http.ssl.enabled: false",
+            "xpack.security.transport.ssl.enabled: false",
+            "xpack.security.enabled: false"
+        );
         Path yamlFile = installation.config("elasticsearch.yml");
-        try (Stream<String> lines = Files.readAllLines(yamlFile).stream()) {
-            if (lines.noneMatch(s -> s.startsWith("xpack.security.enabled"))) {
-                Files.write(yamlFile, configLines, CREATE, APPEND);
-            }
+        List<String> lines;
+        try (Stream<String> allLines = Files.readAllLines(yamlFile).stream()) {
+            lines = allLines.filter(l -> l.startsWith("xpack.security.http.ssl") == false)
+                .filter(l -> l.startsWith("xpack.security.transport.ssl") == false)
+                .filter(l -> l.startsWith("xpack.security.enabled:") == false)
+                .collect(Collectors.toList());
         }
+        lines.addAll(disabledSecurityFeatures);
+        Files.write(yamlFile, lines, TRUNCATE_EXISTING);
+
     }
 
     public static void enableSecurityFeatures(Installation installation) throws IOException {
-        Path yml = installation.config("elasticsearch.yml");
-        List<String> lines;
-        try (Stream<String> allLines = Files.readAllLines(yml).stream()) {
-            lines = allLines.filter(s -> s.startsWith("xpack.security.enabled") == false).collect(Collectors.toList());
-        }
-        Files.write(yml, lines, TRUNCATE_EXISTING);
+        removeSettingFromExistingConfiguration(installation, "xpack.security.enabled");
     }
 
     public static void disableSecurityAutoConfiguration(Installation installation) throws IOException {
+        addSettingToExistingConfiguration(installation, "xpack.security.autoconfiguration.enabled", "false");
+    }
+
+    public static void enableSecurityAutoConfiguration(Installation installation) throws IOException {
+        removeSettingFromExistingConfiguration(installation, "xpack.security.autoconfiguration.enabled");
+    }
+
+    public static void addSettingToExistingConfiguration(Installation installation, String setting, String value) throws IOException {
         Path yml = installation.config("elasticsearch.yml");
-        List<String> addedLines = List.of("xpack.security.autoconfiguration.enabled: false");
         List<String> lines;
         try (Stream<String> allLines = Files.readAllLines(yml).stream()) {
-            lines = allLines.filter(s -> s.startsWith("xpack.security.autoconfiguration.enabled") == false).collect(Collectors.toList());
+            lines = allLines.filter(s -> s.startsWith(setting) == false).collect(Collectors.toList());
         }
-        lines.addAll(addedLines);
+        lines.add(setting + ": " + value);
         Files.write(yml, lines, TRUNCATE_EXISTING);
     }
 
-    public static void enableSecurityAutoConfiguration(Installation installation) throws IOException {
+    public static void removeSettingFromExistingConfiguration(Installation installation, String setting) throws IOException {
         Path yml = installation.config("elasticsearch.yml");
-        List<String> addedLines = List.of("xpack.security.autoconfiguration.enabled: true");
         List<String> lines;
         try (Stream<String> allLines = Files.readAllLines(yml).stream()) {
-            lines = allLines.filter(s -> s.startsWith("xpack.security.autoconfiguration.enabled") == false).collect(Collectors.toList());
+            lines = allLines.filter(s -> s.startsWith(setting) == false).collect(Collectors.toList());
         }
-        lines.addAll(addedLines);
         Files.write(yml, lines, TRUNCATE_EXISTING);
     }
+
 }

+ 6 - 1
qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java

@@ -81,6 +81,11 @@ public class Shell {
     }
 
     public void chown(Path path) throws Exception {
+        chown(path, System.getenv("username"));
+    }
+
+    public void chown(Path path, String newOwner) throws Exception {
+        logger.info("Chowning " + path + " to " + newOwner);
         Platforms.onLinux(() -> run("chown -R elasticsearch:elasticsearch " + path));
         Platforms.onWindows(
             () -> run(
@@ -98,7 +103,7 @@ public class Shell {
                         + "  $acl.SetOwner($account); "
                         + "  Set-Acl $_.FullName $acl "
                         + "}",
-                    System.getenv("username"),
+                    newOwner,
                     path,
                     path
                 )

+ 48 - 11
qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java

@@ -14,7 +14,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.client.fluent.Request;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.packaging.util.Distribution;
 import org.elasticsearch.packaging.util.Distribution.Packaging;
 import org.elasticsearch.packaging.util.FileUtils;
@@ -23,6 +25,7 @@ import org.elasticsearch.packaging.util.ServerUtils;
 import org.elasticsearch.packaging.util.Shell;
 
 import java.io.FileNotFoundException;
+import java.nio.file.InvalidPathException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.PosixFileAttributes;
@@ -41,6 +44,7 @@ import static java.nio.file.attribute.PosixFilePermissions.fromString;
 import static org.elasticsearch.packaging.util.FileMatcher.Fileness.Directory;
 import static org.elasticsearch.packaging.util.FileMatcher.p444;
 import static org.elasticsearch.packaging.util.FileMatcher.p555;
+import static org.elasticsearch.packaging.util.FileMatcher.p660;
 import static org.elasticsearch.packaging.util.FileMatcher.p664;
 import static org.elasticsearch.packaging.util.FileMatcher.p770;
 import static org.elasticsearch.packaging.util.FileMatcher.p775;
@@ -64,7 +68,7 @@ public class Docker {
     public static final Shell sh = new Shell();
     public static final DockerShell dockerShell = new DockerShell();
     public static final int STARTUP_SLEEP_INTERVAL_MILLISECONDS = 1000;
-    public static final int STARTUP_ATTEMPTS_MAX = 10;
+    public static final int STARTUP_ATTEMPTS_MAX = 20;
 
     /**
      * Tracks the currently running Docker image. An earlier implementation used a fixed container name,
@@ -153,7 +157,7 @@ public class Docker {
 
         do {
             try {
-                // Give the container a chance to crash out
+                // Give the container enough time for security auto-configuration or a chance to crash out
                 Thread.sleep(STARTUP_SLEEP_INTERVAL_MILLISECONDS);
 
                 // Set COLUMNS so that `ps` doesn't truncate its output
@@ -269,6 +273,29 @@ public class Docker {
         return result.isSuccess();
     }
 
+    /**
+     * Finds a file or dir in the container and returns its path ( in the container ). If there are multiple matches for the given
+     * pattern, only the first is returned.
+     *
+     * @param base The base path in the container to start the search from
+     * @param type The type we're looking for , d for directories or f for files.
+     * @param pattern the pattern (case insensitive) that matches the file/dir name
+     * @return a Path pointing to the file/directory in the container
+     */
+    public static Path findInContainer(Path base, String type, String pattern) throws InvalidPathException {
+        logger.debug("Trying to look for " + pattern + " ( " + type + ") in " + base + " in the container");
+        final String script = "docker exec " + containerId + " find " + base + " -type " + type + " -iname " + pattern;
+        final Shell.Result result = sh.run(script);
+        if (result.isSuccess() && Strings.isNullOrEmpty(result.stdout) == false) {
+            String path = result.stdout;
+            if (path.split(System.lineSeparator()).length > 1) {
+                path = path.split(System.lineSeparator())[1];
+            }
+            return Path.of(path);
+        }
+        return null;
+    }
+
     /**
      * Run privilege escalated shell command on the local file system via a bind mount inside a Docker container.
      * @param shellCmd The shell command to execute on the localPath e.g. `mkdir /containerPath/dir`.
@@ -408,8 +435,11 @@ public class Docker {
                 )
             );
 
-        Stream.of("elasticsearch.yml", "jvm.options", "log4j2.properties", "role_mapping.yml", "roles.yml", "users", "users_roles")
+        Stream.of("jvm.options", "log4j2.properties", "role_mapping.yml", "roles.yml", "users", "users_roles")
             .forEach(configFile -> assertThat(es.config(configFile), file("root", "root", p664)));
+        // We write to the elasticsearch.yml and elasticsearch.keystore in ConfigInitialNode so it gets owned by elasticsearch.
+        assertThat(es.config("elasticsearch.yml"), file("elasticsearch", "root", p664));
+        assertThat(es.config("elasticsearch.keystore"), file("elasticsearch", "root", p660));
 
         Stream.of("LICENSE.txt", "NOTICE.txt", "README.asciidoc")
             .forEach(doc -> assertThat(es.home.resolve(doc), file("root", "root", p444)));
@@ -458,9 +488,8 @@ public class Docker {
         withLogging(() -> ServerUtils.waitForElasticsearch(installation));
     }
 
-    public static void waitForElasticsearch(String status, String index, Installation installation, String username, String password)
-        throws Exception {
-        withLogging(() -> ServerUtils.waitForElasticsearch(status, index, installation, username, password, null));
+    public static void waitForElasticsearch(Installation installation, String username, String password) {
+        waitForElasticsearch(installation, username, password, null);
     }
 
     /**
@@ -469,10 +498,11 @@ public class Docker {
      * @param installation the installation to check
      * @param username the username to authenticate with
      * @param password the password to authenticate with
+     * @param caCert the CA cert to trust
      */
-    public static void waitForElasticsearch(Installation installation, String username, String password) {
+    public static void waitForElasticsearch(Installation installation, String username, String password, Path caCert) {
         try {
-            waitForElasticsearch("green", null, installation, username, password);
+            withLogging(() -> ServerUtils.waitForElasticsearch("green", null, installation, username, password, caCert));
         } catch (Exception e) {
             throw new AssertionError(
                 "Failed to check whether Elasticsearch had started. This could be because "
@@ -525,16 +555,17 @@ public class Docker {
     }
 
     /**
-     * Fetches the resource from the specified {@code path} on {@code http://localhost:9200}, using
+     * Fetches the resource from the specified {@code path} on {@code http(s)://localhost:9200}, using
      * the supplied authentication credentials.
      *
      * @param path the path to fetch
      * @param user the user to authenticate with
      * @param password the password to authenticate with
+     * @param caCert CA cert to trust, if non-null use the https URL
      * @return a parsed JSON response
      * @throws Exception if something goes wrong
      */
-    public static JsonNode getJson(String path, String user, String password) throws Exception {
+    public static JsonNode getJson(String path, String user, String password, @Nullable Path caCert) throws Exception {
         path = Objects.requireNonNull(path, "path can not be null").trim();
         if (path.isEmpty()) {
             throw new IllegalArgumentException("path must be supplied");
@@ -542,7 +573,13 @@ public class Docker {
         if (path.startsWith("/") == false) {
             throw new IllegalArgumentException("path must start with /");
         }
-        final String pluginsResponse = makeRequest(Request.Get("http://localhost:9200" + path), user, password, null);
+
+        final String pluginsResponse;
+        if (caCert == null) {
+            pluginsResponse = makeRequest(Request.Get("http://localhost:9200" + path), user, password, null);
+        } else {
+            pluginsResponse = makeRequest(Request.Get("https://localhost:9200" + path), user, password, caCert);
+        }
 
         ObjectMapper mapper = new ObjectMapper();
 

+ 1 - 0
x-pack/plugin/security/cli/build.gradle

@@ -10,6 +10,7 @@ dependencies {
   compileOnly project(path: xpackModule('core'))
   api "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
   api "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
+  api "commons-io:commons-io:2.5"
   testImplementation("com.google.jimfs:jimfs:${versions.jimfs}") {
     // this is provided by the runtime classpath, from the security project
     exclude group: "com.google.guava", module: "guava"

+ 1 - 0
x-pack/plugin/security/cli/licenses/commons-io-2.5.jar.sha1

@@ -0,0 +1 @@
+2852e6e05fbb95076fc091f6d1780f1f8fe35e0f

+ 202 - 0
x-pack/plugin/security/cli/licenses/commons-io-LICENSE.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

+ 5 - 0
x-pack/plugin/security/cli/licenses/commons-io-NOTICE.txt

@@ -0,0 +1,5 @@
+Apache Commons IO
+Copyright 2002-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).

+ 336 - 249
x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/ConfigInitialNode.java

@@ -8,7 +8,8 @@
 package org.elasticsearch.xpack.security.cli;
 
 import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
+
+import org.apache.commons.io.FileUtils;
 import org.apache.lucene.util.SetOnce;
 import org.bouncycastle.asn1.x509.GeneralName;
 import org.bouncycastle.asn1.x509.GeneralNames;
@@ -18,6 +19,7 @@ import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.cluster.coordination.ClusterBootstrapService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -25,14 +27,16 @@ import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.discovery.SettingsBasedSeedHostsProvider;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.http.HttpTransportSettings;
-import org.elasticsearch.node.NodeRoleSettings;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.xpack.core.XPackSettings;
 
-import javax.security.auth.x500.X500Principal;
 import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -58,8 +62,8 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
-
-import static org.elasticsearch.xpack.security.cli.CertGenUtils.buildDnFromDomain;
+import java.util.stream.Stream;
+import javax.security.auth.x500.X500Principal;
 
 /**
  * Configures a new cluster node, by appending to the elasticsearch.yml, so that it forms a single node cluster with
@@ -67,10 +71,10 @@ import static org.elasticsearch.xpack.security.cli.CertGenUtils.buildDnFromDomai
  * is started. Subsequent nodes can be added to the cluster via the enrollment flow, but this is not used to
  * configure such nodes or to display the necessary configuration (ie the enrollment tokens) for such.
  *
- * This will not run if Security is explicitly configured or if the existing configuration otherwise clashes with the
- * intent of this (i.e. the node is configured so it cannot form a single node cluster).
+ * This will NOT run if Security is explicitly configured or if the existing configuration otherwise clashes with the
+ * intent of this (i.e. the node is configured so it might not form a single node cluster).
  */
-public class ConfigInitialNode extends EnvironmentAwareCommand {
+public final class ConfigInitialNode extends EnvironmentAwareCommand {
 
     public static final String AUTO_CONFIG_ALT_DN = "CN=Elasticsearch security auto-configuration HTTP CA";
     // the transport keystore is also used as a truststore
@@ -87,10 +91,11 @@ public class ConfigInitialNode extends EnvironmentAwareCommand {
     private static final int HTTP_KEY_SIZE = 4096;
     private static final String TLS_CONFIG_DIR_NAME_PREFIX = "tls_auto_config_initial_node_";
 
-    private final OptionSpec<Void> strictOption = parser.accepts("strict", "Error if auto config cannot be performed for any reason");
-
     public ConfigInitialNode() {
         super("Generates all the necessary security configuration for the initial node of a new secure cluster");
+        // This "cli utility" must be invoked EXCLUSIVELY from the node startup script, where it is passed all the
+        // node startup options unfiltered. It cannot consume most of them, but it does need to inspect the `-E` ones.
+        parser.allowsUnrecognizedOptions();
     }
 
     public static void main(String[] args) throws Exception {
@@ -101,73 +106,70 @@ public class ConfigInitialNode extends EnvironmentAwareCommand {
     protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
         // Silently skipping security auto configuration because node considered as restarting.
         for (Path dataPath : env.dataFiles()) {
-            // TODO: Files.list leaks a file handle because the stream is not closed
-            // this effectively doesn't matter since config is run in a separate, short lived, process
-            // but it should be fixed...
-            if (Files.isDirectory(dataPath) && Files.list(dataPath).findAny().isPresent()) {
-                terminal.println(expectedNoopVerbosityLevel(),
+            if (Files.isDirectory(dataPath) && false == isDirEmpty(dataPath)) {
+                terminal.println(Terminal.Verbosity.VERBOSE,
                     "Skipping security auto configuration because it appears that the node is not starting up for the first time.");
-                terminal.println(expectedNoopVerbosityLevel(),
+                terminal.println(Terminal.Verbosity.VERBOSE,
                     "The node might already be part of a cluster and this auto setup utility is designed to configure Security for new " +
                         "clusters only.");
-                if (options.has(strictOption)) {
-                    throw new UserException(ExitCodes.NOOP, null);
-                } else {
-                    return; // silent error because we wish the node to start as usual (skip auto config) during a restart
-                }
+                // we wish the node to start as usual during a restart
+                // but still the exit code should indicate that this has not been run
+                throw new UserException(ExitCodes.NOOP, null);
             }
         }
-        // preflight checks for the files that are going to be changed
-        // Skipping security auto configuration if configuration files cannot be mutated (ie are read-only)
+
+        // pre-flight checks for the files that are going to be changed
         final Path ymlPath = env.configFile().resolve("elasticsearch.yml");
         final Path keystorePath = KeyStoreWrapper.keystorePath(env.configFile());
-        try {
-            // it is odd for the `elasticsearch.yml` file to be missing or not be a regular (the node won't start)
-            // but auto configuration should not be concerned with fixing it (by creating the file) and let the node startup fail
-            if (false == Files.exists(ymlPath) || false == Files.isRegularFile(ymlPath, LinkOption.NOFOLLOW_LINKS)) {
-                terminal.println(unexpectedNoopVerbosityLevel(), String.format(Locale.ROOT, "Skipping security auto configuration because" +
-                        " the configuration file [%s] is missing or is not a regular file", ymlPath));
-                throw new UserException(ExitCodes.CONFIG, null);
-            }
-            // If the node's yml configuration is not readable, most probably auto-configuration isn't run under the suitable user
-            if (false == Files.isReadable(ymlPath)) {
-                terminal.println(unexpectedNoopVerbosityLevel(), String.format(Locale.ROOT, "Skipping security auto configuration because" +
-                        " the configuration file [%s] is not readable", ymlPath));
-                throw new UserException(ExitCodes.NOOP, null);
-            }
-            // Inform that auto-configuration will not run if keystore cannot be read.
-            if (Files.exists(keystorePath) && (false == Files.isRegularFile(keystorePath, LinkOption.NOFOLLOW_LINKS) ||
-                    false == Files.isReadable(keystorePath))) {
-                terminal.println(unexpectedNoopVerbosityLevel(), String.format(Locale.ROOT, "Skipping security auto configuration because" +
-                        " the node keystore file [%s] is not a readable regular file", keystorePath));
-                throw new UserException(ExitCodes.NOOP, null);
-            }
-        } catch (UserException e) {
-            if (options.has(strictOption)) {
-                throw e;
-            } else {
-                return; // silent error because we wish the node to start as usual (skip auto config) if the configuration is read-only
-            }
+        // it is odd for the `elasticsearch.yml` file to be missing or not be a regular (the node won't start)
+        // but auto configuration should not be concerned with fixing it (by creating the file) and let the node startup fail
+        if (false == Files.exists(ymlPath) || false == Files.isRegularFile(ymlPath, LinkOption.NOFOLLOW_LINKS)) {
+            terminal.println(
+                Terminal.Verbosity.NORMAL,
+                String.format(
+                    Locale.ROOT,
+                    "Skipping security auto configuration because the configuration file [%s] is missing or is not a regular file",
+                    ymlPath
+                )
+            );
+            throw new UserException(ExitCodes.CONFIG, null);
+        }
+        // If the node's yml configuration is not readable, most probably auto-configuration isn't run under the suitable user
+        if (false == Files.isReadable(ymlPath)) {
+            terminal.println(
+                Terminal.Verbosity.NORMAL,
+                String.format(
+                    Locale.ROOT,
+                    "Skipping security auto configuration because the current user does not have permission to read "
+                        + " configuration file [%s]",
+                    ymlPath
+                )
+            );
+            throw new UserException(ExitCodes.NOOP, null);
+        }
+        // Inform that auto-configuration will not run if keystore cannot be read.
+        if (Files.exists(keystorePath)
+            && (false == Files.isRegularFile(keystorePath, LinkOption.NOFOLLOW_LINKS) || false == Files.isReadable(keystorePath))) {
+            terminal.println(
+                Terminal.Verbosity.NORMAL,
+                String.format(
+                    Locale.ROOT,
+                    "Skipping security auto configuration because the node keystore file [%s] is not a readable regular file",
+                    keystorePath
+                )
+            );
+            throw new UserException(ExitCodes.NOOP, null);
         }
 
         // only perform auto-configuration if the existing configuration is not conflicting (eg Security already enabled)
         // if it is, silently skip auto configuration
-        try {
-            checkExistingConfiguration(env, terminal);
-        } catch (UserException e) {
-            if (options.has(strictOption)) {
-                throw e;
-            } else {
-                return; // silent error because we wish the node to start as usual (skip auto config) if certain configurations are set
-            }
-        }
+        checkExistingConfiguration(env.settings(), terminal);
 
         final ZonedDateTime autoConfigDate = ZonedDateTime.now(ZoneOffset.UTC);
-        final String instantAutoConfigName = TLS_CONFIG_DIR_NAME_PREFIX + autoConfigDate.toInstant().getEpochSecond();;
+        final String instantAutoConfigName = TLS_CONFIG_DIR_NAME_PREFIX + autoConfigDate.toInstant().getEpochSecond();
         final Path instantAutoConfigDir = env.configFile().resolve(instantAutoConfigName);
         try {
             // it is useful to pre-create the sub-config dir in order to check that the config dir is writable and that file owners match
-            // THIS AUTO CONFIGURATION COMMAND WILL NOT CHANGE THE OWNERS OF CONFIG FILES
             Files.createDirectory(instantAutoConfigDir);
             // set permissions to 750, don't rely on umask, we assume auto configuration preserves ownership so we don't have to
             // grant "group" or "other" permissions
@@ -175,112 +177,155 @@ public class ConfigInitialNode extends EnvironmentAwareCommand {
             if (view != null) {
                 view.setPermissions(PosixFilePermissions.fromString("rwxr-x---"));
             }
-        } catch (Exception e) {
+        } catch (Throwable t) {
             try {
-                Files.deleteIfExists(instantAutoConfigDir);
+                deleteDirectory(instantAutoConfigDir);
             } catch (Exception ex) {
-                e.addSuppressed(ex);
+                t.addSuppressed(ex);
             }
             // the config dir is probably read-only, either because this auto-configuration runs as a different user from the install user,
             // or if the admin explicitly makes configuration immutable (read-only), both of which are reasons to skip auto-configuration
             // this will show a message to the console (the first time the node starts) and auto-configuration is effectively bypassed
             // the message will not be subsequently shown (because auto-configuration doesn't run for node restarts)
-            if (options.has(strictOption)) {
-                throw new UserException(ExitCodes.CANT_CREATE, "Could not create auto configuration directory", e);
-            } else {
-                return; // silent error because we wish the node to start as usual (skip auto config) if config dir is not writable
-            }
+            throw new UserException(ExitCodes.CANT_CREATE, "Could not create auto configuration directory", t);
         }
 
-        // Ensure that the files created by the auto-config command MUST have the same owner as the config dir itself,
-        // as well as that the replaced files don't change ownership.
-        // This is because the files created by this command have hard-coded "no" permissions for "group" and "other"
-        UserPrincipal newFileOwner = Files.getOwner(instantAutoConfigDir, LinkOption.NOFOLLOW_LINKS);
-        if ((false == newFileOwner.equals(Files.getOwner(env.configFile(), LinkOption.NOFOLLOW_LINKS))) ||
-                (false == newFileOwner.equals(Files.getOwner(ymlPath, LinkOption.NOFOLLOW_LINKS))) ||
-                (Files.exists(keystorePath) && false == newFileOwner.equals(Files.getOwner(keystorePath, LinkOption.NOFOLLOW_LINKS)))) {
-            Files.deleteIfExists(instantAutoConfigDir);
-            if (options.has(strictOption)) {
-                throw new UserException(ExitCodes.CONFIG, "Aborting auto configuration because it would change config file owners");
-            } else {
-                return; // if a different user runs ES compared to the user that installed it, auto configuration will not run
-            }
+        // Check that the created auto-config dir has the same owner as the config dir.
+        // This is a sort of sanity check.
+        // If the node process works OK given the owner of the config dir, it should also tolerate the auto-created config dir,
+        // provided that they both have the same owner and permissions.
+        final UserPrincipal newFileOwner = Files.getOwner(instantAutoConfigDir, LinkOption.NOFOLLOW_LINKS);
+        if (false == newFileOwner.equals(Files.getOwner(env.configFile(), LinkOption.NOFOLLOW_LINKS))) {
+            deleteDirectory(instantAutoConfigDir);
+            // the following is only printed once, if the node starts successfully
+            throw new UserException(
+                ExitCodes.CONFIG,
+                "Aborting auto configuration because of config dir ownership mismatch. Config dir is owned by "
+                    + Files.getOwner(env.configFile(), LinkOption.NOFOLLOW_LINKS).getName()
+                    + " but auto-configuration directory would be owned by "
+                    + newFileOwner.getName()
+            );
         }
+        final KeyPair transportKeyPair;
+        final X509Certificate transportCert;
+        final KeyPair httpCAKeyPair;
+        final X509Certificate httpCACert;
+        final KeyPair httpKeyPair;
+        final X509Certificate httpCert;
+        try {
+            // the transport key-pair is the same across the cluster and is trusted without hostname verification (it is self-signed),
+            final X500Principal certificatePrincipal = new X500Principal("CN=" + System.getenv("HOSTNAME"));
+            final X500Principal caPrincipal = new X500Principal(AUTO_CONFIG_ALT_DN);
+            // this does DNS resolve and could block
+            final GeneralNames subjectAltNames = getSubjectAltNames();
 
-        // the transport key-pair is the same across the cluster and is trusted without hostname verification (it is self-signed),
-        final X500Principal certificatePrincipal = new X500Principal(buildDnFromDomain(System.getenv("HOSTNAME")));
-        final X500Principal caPrincipal = new X500Principal(AUTO_CONFIG_ALT_DN);
-        // this does DNS resolve and could block
-        final GeneralNames subjectAltNames = getSubjectAltNames();
+            transportKeyPair = CertGenUtils.generateKeyPair(TRANSPORT_KEY_SIZE);
+            // self-signed which is not a CA
+            transportCert = CertGenUtils.generateSignedCertificate(
+                certificatePrincipal,
+                subjectAltNames,
+                transportKeyPair,
+                null,
+                null,
+                false,
+                TRANSPORT_CERTIFICATE_DAYS,
+                "SHA256withRSA"
+            );
+            httpCAKeyPair = CertGenUtils.generateKeyPair(HTTP_CA_KEY_SIZE);
+            // self-signed CA
+            httpCACert = CertGenUtils.generateSignedCertificate(
+                caPrincipal,
+                null,
+                httpCAKeyPair,
+                null,
+                null,
+                true,
+                HTTP_CA_CERTIFICATE_DAYS,
+                "SHA256withRSA"
+            );
+            httpKeyPair = CertGenUtils.generateKeyPair(HTTP_KEY_SIZE);
+            // non-CA
+            httpCert = CertGenUtils.generateSignedCertificate(
+                certificatePrincipal,
+                subjectAltNames,
+                httpKeyPair,
+                httpCACert,
+                httpCAKeyPair.getPrivate(),
+                false,
+                HTTP_CERTIFICATE_DAYS,
+                "SHA256withRSA"
+            );
 
-        KeyPair transportKeyPair = CertGenUtils.generateKeyPair(TRANSPORT_KEY_SIZE);
-        // self-signed which is not a CA
-        X509Certificate transportCert = CertGenUtils.generateSignedCertificate(certificatePrincipal,
-            subjectAltNames, transportKeyPair, null, null, false, TRANSPORT_CERTIFICATE_DAYS, "SHA256withRSA");
-        KeyPair httpCAKeyPair = CertGenUtils.generateKeyPair(HTTP_CA_KEY_SIZE);
-        // self-signed CA
-        X509Certificate httpCACert = CertGenUtils.generateSignedCertificate(caPrincipal,
-            null , httpCAKeyPair, null, null, true, HTTP_CA_CERTIFICATE_DAYS, "SHA256withRSA");
-        KeyPair httpKeyPair = CertGenUtils.generateKeyPair(HTTP_KEY_SIZE);
-        // non-CA
-        X509Certificate httpCert = CertGenUtils.generateSignedCertificate(certificatePrincipal,
-            subjectAltNames, httpKeyPair, httpCACert, httpCAKeyPair.getPrivate(), false, HTTP_CERTIFICATE_DAYS, "SHA256withRSA");
+            // the HTTP CA PEM file is provided "just in case". The node doesn't use it, but clients (configured manually, outside of the
+            // enrollment process) might indeed need it, and it is currently impossible to retrieve it
 
-        // the HTTP CA PEM file is provided "just in case", the node configuration doesn't use it
-        // but clients (configured manually, outside of the enrollment process) might indeed need it and
-        // it is impossible to use the keystore because it is password protected because it contains the key
-        try {
             fullyWriteFile(instantAutoConfigDir, HTTP_AUTOGENERATED_CA_NAME + ".crt", false, stream -> {
-                try (JcaPEMWriter pemWriter =
-                             new JcaPEMWriter(new BufferedWriter(new OutputStreamWriter(stream, StandardCharsets.UTF_8)))) {
+                try (
+                    JcaPEMWriter pemWriter = new JcaPEMWriter(new BufferedWriter(new OutputStreamWriter(stream, StandardCharsets.UTF_8)))
+                ) {
                     pemWriter.writeObject(httpCACert);
                 }
             });
-        } catch (Exception e) {
-            Files.deleteIfExists(instantAutoConfigDir);
-            throw e; // this is an error which mustn't be ignored during node startup
+        } catch (Throwable t) {
+            deleteDirectory(instantAutoConfigDir);
+            // this is an error which mustn't be ignored during node startup
+            // the exit code for unhandled Exceptions is "1"
+            throw t;
         }
 
         // save original keystore before updating (replacing)
-        final Path keystoreBackupPath =
-                env.configFile().resolve(KeyStoreWrapper.KEYSTORE_FILENAME + "." + autoConfigDate.toInstant().getEpochSecond() + ".orig");
+        final Path keystoreBackupPath = env.configFile()
+            .resolve(KeyStoreWrapper.KEYSTORE_FILENAME + "." + autoConfigDate.toInstant().getEpochSecond() + ".orig");
         if (Files.exists(keystorePath)) {
             try {
                 Files.copy(keystorePath, keystoreBackupPath, StandardCopyOption.COPY_ATTRIBUTES);
-            } catch (Exception e) {
+            } catch (Throwable t) {
                 try {
-                    Files.deleteIfExists(instantAutoConfigDir);
+                    deleteDirectory(instantAutoConfigDir);
                 } catch (Exception ex) {
-                    e.addSuppressed(ex);
+                    t.addSuppressed(ex);
                 }
-                throw e;
+                throw t;
             }
         }
 
         final SetOnce<SecureString> nodeKeystorePassword = new SetOnce<>();
         try (KeyStoreWrapper nodeKeystore = KeyStoreWrapper.bootstrap(env.configFile(), () -> {
-            nodeKeystorePassword.set(new SecureString(terminal.readSecret(nodeKeystorePasswordPrompt(),
-                    KeyStoreWrapper.MAX_PASSPHRASE_LENGTH)));
+            nodeKeystorePassword.set(new SecureString(terminal.readSecret("", KeyStoreWrapper.MAX_PASSPHRASE_LENGTH)));
             return nodeKeystorePassword.get().clone();
         })) {
             // do not overwrite keystore entries
             // instead expect the user to manually remove them herself
-            if (nodeKeystore.getSettingNames().contains("xpack.security.transport.ssl.keystore.secure_password") ||
-                nodeKeystore.getSettingNames().contains("xpack.security.transport.ssl.truststore.secure_password") ||
-                nodeKeystore.getSettingNames().contains("xpack.security.http.ssl.keystore.secure_password")) {
-                throw new UserException(ExitCodes.CONFIG, "Aborting auto configuration because the node keystore contains password " +
-                        "settings already"); // it is OK to silently ignore these because the node won't start
+            if (nodeKeystore.getSettingNames().contains("xpack.security.transport.ssl.keystore.secure_password")
+                || nodeKeystore.getSettingNames().contains("xpack.security.transport.ssl.truststore.secure_password")
+                || nodeKeystore.getSettingNames().contains("xpack.security.http.ssl.keystore.secure_password")) {
+                // this error condition is akin to condition of existing configuration in the yml file
+                // this is not a fresh install and the admin has something planned for Security
+                // Even though this is probably invalid configuration, do NOT fix it, let the node fail to start in its usual way.
+                // Still display a message, because this can be tricky to figure out (why auto-conf did not run) if by mistake.
+                throw new UserException(
+                    ExitCodes.CONFIG,
+                    "Aborting auto configuration because the node keystore contains password " + "settings already"
+                );
             }
             try (SecureString transportKeystorePassword = newKeystorePassword()) {
                 KeyStore transportKeystore = KeyStore.getInstance("PKCS12");
                 transportKeystore.load(null);
                 // the PKCS12 keystore and the contained private key use the same password
-                transportKeystore.setKeyEntry(TRANSPORT_AUTOGENERATED_KEY_ALIAS, transportKeyPair.getPrivate(),
-                    transportKeystorePassword.getChars(), new Certificate[]{transportCert});
+                transportKeystore.setKeyEntry(
+                    TRANSPORT_AUTOGENERATED_KEY_ALIAS,
+                    transportKeyPair.getPrivate(),
+                    transportKeystorePassword.getChars(),
+                    new Certificate[] { transportCert }
+                );
                 // the transport keystore is used as a trustore too, hence it must contain a certificate entry
                 transportKeystore.setCertificateEntry(TRANSPORT_AUTOGENERATED_CERT_ALIAS, transportCert);
-                fullyWriteFile(instantAutoConfigDir, TRANSPORT_AUTOGENERATED_KEYSTORE_NAME + ".p12", false,
-                    stream -> transportKeystore.store(stream, transportKeystorePassword.getChars()));
+                fullyWriteFile(
+                    instantAutoConfigDir,
+                    TRANSPORT_AUTOGENERATED_KEYSTORE_NAME + ".p12",
+                    false,
+                    stream -> transportKeystore.store(stream, transportKeystorePassword.getChars())
+                );
                 nodeKeystore.setString("xpack.security.transport.ssl.keystore.secure_password", transportKeystorePassword.getChars());
                 // we use the same PKCS12 file for the keystore and the truststore
                 nodeKeystore.setString("xpack.security.transport.ssl.truststore.secure_password", transportKeystorePassword.getChars());
@@ -290,41 +335,45 @@ public class ConfigInitialNode extends EnvironmentAwareCommand {
                 httpKeystore.load(null);
                 // the keystore contains both the node's and the CA's private keys
                 // both keys are encrypted using the same password as the PKCS12 keystore they're contained in
-                httpKeystore.setKeyEntry(HTTP_AUTOGENERATED_KEYSTORE_NAME + "_ca", httpCAKeyPair.getPrivate(),
-                    httpKeystorePassword.getChars(), new Certificate[]{httpCACert});
-                httpKeystore.setKeyEntry(HTTP_AUTOGENERATED_KEYSTORE_NAME, httpKeyPair.getPrivate(),
-                    httpKeystorePassword.getChars(), new Certificate[]{httpCert, httpCACert});
-                fullyWriteFile(instantAutoConfigDir, HTTP_AUTOGENERATED_KEYSTORE_NAME + ".p12", false,
-                    stream -> httpKeystore.store(stream, httpKeystorePassword.getChars()));
+                httpKeystore.setKeyEntry(
+                    HTTP_AUTOGENERATED_KEYSTORE_NAME + "_ca",
+                    httpCAKeyPair.getPrivate(),
+                    httpKeystorePassword.getChars(),
+                    new Certificate[] { httpCACert }
+                );
+                httpKeystore.setKeyEntry(
+                    HTTP_AUTOGENERATED_KEYSTORE_NAME,
+                    httpKeyPair.getPrivate(),
+                    httpKeystorePassword.getChars(),
+                    new Certificate[] { httpCert, httpCACert }
+                );
+                fullyWriteFile(
+                    instantAutoConfigDir,
+                    HTTP_AUTOGENERATED_KEYSTORE_NAME + ".p12",
+                    false,
+                    stream -> httpKeystore.store(stream, httpKeystorePassword.getChars())
+                );
                 nodeKeystore.setString("xpack.security.http.ssl.keystore.secure_password", httpKeystorePassword.getChars());
             }
             // finally overwrites the node keystore (if the keystores have been successfully written)
             nodeKeystore.save(env.configFile(), nodeKeystorePassword.get() == null ? new char[0] : nodeKeystorePassword.get().getChars());
-        } catch (Exception e) {
+        } catch (Throwable t) {
             // restore keystore to revert possible keystore bootstrap
             try {
                 if (Files.exists(keystoreBackupPath)) {
-                    Files.move(keystoreBackupPath, keystorePath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE,
-                            StandardCopyOption.COPY_ATTRIBUTES);
+                    Files.move(keystoreBackupPath, keystorePath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
                 } else {
                     Files.deleteIfExists(keystorePath);
                 }
             } catch (Exception ex) {
-                e.addSuppressed(ex);
+                t.addSuppressed(ex);
             }
             try {
-                Files.deleteIfExists(instantAutoConfigDir);
+                deleteDirectory(instantAutoConfigDir);
             } catch (Exception ex) {
-                e.addSuppressed(ex);
-            }
-            if (false == (e instanceof UserException)) {
-                throw e; // unexpected exections should prevent the node from starting
-            }
-            if (options.has(strictOption)) {
-                throw e;
-            } else {
-                return; // ignoring if the keystore contains password values already, so that the node startup deals with it (fails)
+                t.addSuppressed(ex);
             }
+            throw t;
         } finally {
             if (nodeKeystorePassword.get() != null) {
                 nodeKeystorePassword.get().close();
@@ -365,7 +414,9 @@ public class ConfigInitialNode extends EnvironmentAwareCommand {
                     bw.write(XPackSettings.SECURITY_ENABLED.getKey() + ": true");
                     bw.newLine();
                     bw.newLine();
-                    if (false == env.settings().hasValue(XPackSettings.ENROLLMENT_ENABLED.getKey())) {
+                    // Set enrollment mode to true unless user explicitly set it to false themselves
+                    if (false == (env.settings().hasValue(XPackSettings.ENROLLMENT_ENABLED.getKey())
+                        && false == XPackSettings.ENROLLMENT_ENABLED.get(env.settings()))) {
                         bw.write(XPackSettings.ENROLLMENT_ENABLED.getKey() + ": true");
                         bw.newLine();
                         bw.newLine();
@@ -377,73 +428,101 @@ public class ConfigInitialNode extends EnvironmentAwareCommand {
                     bw.newLine();
                     bw.write("xpack.security.transport.ssl.verification_mode: certificate");
                     bw.newLine();
-                    bw.write("xpack.security.transport.ssl.keystore.path: " + instantAutoConfigDir
-                            .resolve(TRANSPORT_AUTOGENERATED_KEYSTORE_NAME + ".p12"));
+                    bw.write(
+                        "xpack.security.transport.ssl.keystore.path: "
+                            + instantAutoConfigDir.resolve(TRANSPORT_AUTOGENERATED_KEYSTORE_NAME + ".p12")
+                    );
                     bw.newLine();
                     // we use the keystore as a truststore in order to minimize the number of auto-generated resources,
                     // and also because a single file is more idiomatic to the scheme of a shared secret between the cluster nodes
                     // no one should only need the TLS cert without the associated key for the transport layer
-                    bw.write("xpack.security.transport.ssl.truststore.path: " + instantAutoConfigDir
-                            .resolve(TRANSPORT_AUTOGENERATED_KEYSTORE_NAME + ".p12"));
+                    bw.write(
+                        "xpack.security.transport.ssl.truststore.path: "
+                            + instantAutoConfigDir.resolve(TRANSPORT_AUTOGENERATED_KEYSTORE_NAME + ".p12")
+                    );
                     bw.newLine();
 
                     bw.newLine();
                     bw.write("xpack.security.http.ssl.enabled: true");
                     bw.newLine();
-                    bw.write("xpack.security.http.ssl.keystore.path: " + instantAutoConfigDir.resolve(HTTP_AUTOGENERATED_KEYSTORE_NAME +
-                            ".p12"));
+                    bw.write(
+                        "xpack.security.http.ssl.keystore.path: " + instantAutoConfigDir.resolve(HTTP_AUTOGENERATED_KEYSTORE_NAME + ".p12")
+                    );
                     bw.newLine();
 
+                    // we have configured TLS on the transport layer with newly generated certs and keys,
+                    // hence this node cannot form a multi-node cluster
+                    // if we don't set the following the node might trip the discovery bootstrap check
+                    if (false == DiscoveryModule.isSingleNodeDiscovery(env.settings())
+                        && false == ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.exists(env.settings())) {
+                        bw.newLine();
+                        bw.write("# The initial node with security auto-configured must form a cluster on its own,");
+                        bw.newLine();
+                        bw.write("# and all the subsequent nodes should be added via the node enrollment flow");
+                        bw.newLine();
+                        bw.write(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey() + ": [\"${HOSTNAME}\"]");
+                        bw.newLine();
+                    }
+
                     // if any address settings have been set, assume the admin has thought it through wrt to addresses,
                     // and don't try to be smart and mess with that
-                    if (false == (env.settings().hasValue(HttpTransportSettings.SETTING_HTTP_HOST.getKey()) ||
-                            env.settings().hasValue(HttpTransportSettings.SETTING_HTTP_BIND_HOST.getKey()) ||
-                            env.settings().hasValue(HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST.getKey()) ||
-                            env.settings().hasValue(NetworkService.GLOBAL_NETWORK_HOST_SETTING.getKey()) ||
-                            env.settings().hasValue(NetworkService.GLOBAL_NETWORK_BIND_HOST_SETTING.getKey()) ||
-                            env.settings().hasValue(NetworkService.GLOBAL_NETWORK_PUBLISH_HOST_SETTING.getKey()))) {
+                    if (false == (env.settings().hasValue(HttpTransportSettings.SETTING_HTTP_HOST.getKey())
+                        || env.settings().hasValue(HttpTransportSettings.SETTING_HTTP_BIND_HOST.getKey())
+                        || env.settings().hasValue(HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST.getKey())
+                        || env.settings().hasValue(NetworkService.GLOBAL_NETWORK_HOST_SETTING.getKey())
+                        || env.settings().hasValue(NetworkService.GLOBAL_NETWORK_BIND_HOST_SETTING.getKey())
+                        || env.settings().hasValue(NetworkService.GLOBAL_NETWORK_PUBLISH_HOST_SETTING.getKey()))) {
                         bw.newLine();
-                        bw.write("# With security now configured, which includes user authentication over HTTPs, " +
-                                "it's reasonable to serve requests on the local network too");
+                        bw.write(
+                            "# With security now configured, which includes user authentication over HTTPs, "
+                                + "it's reasonable to serve requests on the local network too"
+                        );
                         bw.newLine();
                         bw.write(HttpTransportSettings.SETTING_HTTP_HOST.getKey() + ": [_local_, _site_]");
                         bw.newLine();
                     }
                 }
             });
-        } catch (Exception e) {
+        } catch (Throwable t) {
             try {
                 if (Files.exists(keystoreBackupPath)) {
-                    Files.move(keystoreBackupPath, keystorePath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE,
-                            StandardCopyOption.COPY_ATTRIBUTES);
+                    Files.move(
+                        keystoreBackupPath,
+                        keystorePath,
+                        StandardCopyOption.REPLACE_EXISTING,
+                        StandardCopyOption.ATOMIC_MOVE,
+                        StandardCopyOption.COPY_ATTRIBUTES
+                    );
                 } else {
                     Files.deleteIfExists(keystorePath);
                 }
             } catch (Exception ex) {
-                e.addSuppressed(ex);
+                t.addSuppressed(ex);
             }
             try {
-                Files.deleteIfExists(instantAutoConfigDir);
+                deleteDirectory(instantAutoConfigDir);
             } catch (Exception ex) {
-                e.addSuppressed(ex);
+                t.addSuppressed(ex);
             }
-            throw e;
+            throw t;
         }
+        // only delete the backed up file if all went well
         Files.deleteIfExists(keystoreBackupPath);
     }
 
-    @SuppressForbidden(reason = "InetAddress#getCanonicalHostName used to populate auto generated HTTPS cert")
+    @SuppressForbidden(reason = "Uses File API because the commons io library does, which is useful for file manipulation")
+    private void deleteDirectory(Path directory) throws IOException {
+        FileUtils.deleteDirectory(directory.toFile());
+    }
+
     private GeneralNames getSubjectAltNames() throws IOException {
         Set<GeneralName> generalNameSet = new HashSet<>();
         for (InetAddress ip : NetworkUtils.getAllAddresses()) {
             String ipString = NetworkAddress.format(ip);
             generalNameSet.add(new GeneralName(GeneralName.iPAddress, ipString));
-            String reverseFQDN = ip.getCanonicalHostName();
-            if (false == ipString.equals(reverseFQDN)) {
-                // reverse FQDN successful
-                generalNameSet.add(new GeneralName(GeneralName.dNSName, reverseFQDN));
-            }
         }
+        generalNameSet.add(new GeneralName(GeneralName.dNSName, "localhost"));
+        generalNameSet.add(new GeneralName(GeneralName.dNSName, System.getenv("HOSTNAME")));
         return new GeneralNames(generalNameSet.toArray(new GeneralName[0]));
     }
 
@@ -452,88 +531,102 @@ public class ConfigInitialNode extends EnvironmentAwareCommand {
         return UUIDs.randomBase64UUIDSecureString();
     }
 
-    // Detect if the existing yml configuration is incompatible with auto-configuration,
-    // in which case auto-configuration is SILENTLY skipped.
-    // This assumes the user knows what she's doing when configuring the node.
-    void checkExistingConfiguration(Environment environment, Terminal terminal) throws UserException {
-        // Silently skipping security auto configuration, because Security is already configured.
-        if (environment.settings().hasValue(XPackSettings.SECURITY_ENABLED.getKey())) {
-            // do not try to validate, correct or fill in any incomplete security configuration,
-            // instead rely on the regular node startup to do this validation
-            terminal.println(expectedNoopVerbosityLevel(),
-                    "Skipping security auto configuration because it appears that security is already configured.");
+    /*
+     * Detect if the existing yml configuration is incompatible with auto-configuration,
+     * in which case auto-configuration is SILENTLY skipped.
+     * This assumes the user knows what they are doing when configuring the node.
+     */
+    void checkExistingConfiguration(Settings settings, Terminal terminal) throws UserException {
+        // Allow the user to explicitly set that they don't want auto-configuration for security, regardless of our heuristics
+        if (XPackSettings.SECURITY_AUTOCONFIGURATION_ENABLED.get(settings) == false) {
+            terminal.println(
+                Terminal.Verbosity.VERBOSE,
+                "Skipping security auto configuration because [" + XPackSettings.SECURITY_AUTOCONFIGURATION_ENABLED.getKey() + "] is false"
+            );
             throw new UserException(ExitCodes.NOOP, null);
         }
-        // Silently skipping security auto configuration if enrollment is disabled.
-        // But tolerate enrollment explicitly enabled, as it could be useful to enable it by a command line option
-        // only the first time that the node is started.
-        if (environment.settings().hasValue(XPackSettings.ENROLLMENT_ENABLED.getKey()) && false ==
-                XPackSettings.ENROLLMENT_ENABLED.get(environment.settings())) {
-            terminal.println(expectedNoopVerbosityLevel(),
-                    "Skipping security auto configuration because enrollment is explicitly disabled.");
+        // Silently skip security auto configuration when Security is already configured.
+        // Security is enabled implicitly, but if the admin chooses to enable it explicitly then
+        // skip the TLS auto-configuration, as this is a sign that the admin is opting for the default behavior
+        if (XPackSettings.SECURITY_ENABLED.exists(settings)) {
+            // do not try to validate, correct or fill in any incomplete security configuration,
+            // instead rely on the regular node startup to do this validation
+            terminal.println(
+                Terminal.Verbosity.VERBOSE,
+                "Skipping security auto configuration because it appears that security is already configured."
+            );
             throw new UserException(ExitCodes.NOOP, null);
         }
-        // Silently skipping security auto configuration because the node is configured for cluster formation.
-        // Auto-configuration assumes that this is done in order to configure a multi-node cluster,
-        // and Security auto-configuration doesn't work when bootstrapping a multi node clusters
-        if (environment.settings().hasValue(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey())) {
-            terminal.println(expectedNoopVerbosityLevel(),
-                    "Skipping security auto configuration because this node is explicitly configured to form a new cluster.");
-            terminal.println(expectedNoopVerbosityLevel(),
-                    "The node cannot be auto configured to participate in forming a new multi-node secure cluster.");
+        // Security auto configuration must not run if the node is configured for multi-node cluster formation (bootstrap or join).
+        // This is because transport TLS with newly generated certs will hinder cluster formation because the other nodes cannot trust yet.
+        if (false == isInitialClusterNode(settings)) {
+            terminal.println(
+                Terminal.Verbosity.VERBOSE,
+                "Skipping security auto configuration because this node is configured to bootstrap or to join a "
+                    + "multi-node cluster, which is not supported."
+            );
             throw new UserException(ExitCodes.NOOP, null);
         }
-        // Silently skipping security auto configuration because node cannot become master.
-        final List<DiscoveryNodeRole> nodeRoles = NodeRoleSettings.NODE_ROLES_SETTING.get(environment.settings());
-        boolean canBecomeMaster = nodeRoles.contains(DiscoveryNodeRole.MASTER_ROLE) &&
-                false == nodeRoles.contains(DiscoveryNodeRole.VOTING_ONLY_NODE_ROLE);
+        // Silently skip security auto configuration because node cannot become master.
+        boolean canBecomeMaster = DiscoveryNode.isMasterNode(settings)
+            && false == DiscoveryNode.hasRole(settings, DiscoveryNodeRole.VOTING_ONLY_NODE_ROLE);
         if (false == canBecomeMaster) {
-            terminal.println(expectedNoopVerbosityLevel(),
-                    "Skipping security auto configuration because the node is configured such that it cannot become master.");
+            terminal.println(
+                Terminal.Verbosity.VERBOSE,
+                "Skipping security auto configuration because the node is configured such that it cannot become master."
+            );
             throw new UserException(ExitCodes.NOOP, null);
         }
-        // Silently skipping security auto configuration, because the node cannot contain the Security index data
-        boolean canHoldSecurityIndex = nodeRoles.stream().anyMatch(DiscoveryNodeRole::canContainData);
+        // Silently skip security auto configuration, because the node cannot contain the Security index data
+        boolean canHoldSecurityIndex = DiscoveryNode.canContainData(settings);
         if (false == canHoldSecurityIndex) {
-            terminal.println(expectedNoopVerbosityLevel(),
-                    "Skipping security auto configuration because the node is configured such that it cannot contain data.");
+            terminal.println(
+                Terminal.Verbosity.VERBOSE,
+                "Skipping security auto configuration because the node is configured such that it cannot contain data."
+            );
             throw new UserException(ExitCodes.NOOP, null);
         }
         // Silently skipping security auto configuration because TLS is already configured
-        if (false == environment.settings().getByPrefix(XPackSettings.TRANSPORT_SSL_PREFIX).isEmpty() ||
-                false == environment.settings().getByPrefix(XPackSettings.HTTP_SSL_PREFIX).isEmpty()) {
-            // zero validation for the TLS settings as well, let the node bootup do its thing
-            terminal.println(expectedNoopVerbosityLevel(),
-                    "Skipping security auto configuration because it appears that TLS is already configured.");
+        if (false == settings.getByPrefix(XPackSettings.TRANSPORT_SSL_PREFIX).isEmpty()
+            || false == settings.getByPrefix(XPackSettings.HTTP_SSL_PREFIX).isEmpty()) {
+            // zero validation for the TLS settings as well, let the node boot and do its thing
+            terminal.println(
+                Terminal.Verbosity.VERBOSE,
+                "Skipping security auto configuration because it appears that TLS is already configured."
+            );
             throw new UserException(ExitCodes.NOOP, null);
         }
-        // auto-configuration runs even if the realms are configured in any way (assuming defining realms is permitted without touching
-        // the xpack.security.enabled setting, otherwise auto-config doesn't run, see previous condition)
+        // auto-configuration runs even if the realms are configured in any way,
+        // including defining file based users (defining realms is permitted without touching
+        // the xpack.security.enabled setting)
         // but the file realm is required for some of the auto-configuration parts (setting/resetting the elastic user)
         // if disabled, it must be manually enabled back and, preferably, at the head of the realm chain
     }
 
-    String nodeKeystorePasswordPrompt() {
-        return "Enter password for the elasticsearch keystore : ";
-    }
-
-    Terminal.Verbosity expectedNoopVerbosityLevel() {
-        return Terminal.Verbosity.NORMAL;
+    // Unfortunately, we cannot tell, for every configuration, if it is going to result in a multi node cluster, as it depends
+    // on the addresses that this node, and the others, will bind to when starting (and this runs on a single node before it
+    // starts).
+    // Here we take a conservative approach: if any of the discovery or initial master nodes setting are set to a non-empty
+    // value, we assume the admin intended a multi-node cluster configuration. There is only one exception: if the initial master
+    // nodes setting contains just the current node name.
+    private boolean isInitialClusterNode(Settings settings) {
+        return DiscoveryModule.isSingleNodeDiscovery(settings)
+            || (ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.get(settings).isEmpty()
+                && SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING.get(settings).isEmpty()
+                && DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.get(settings).isEmpty())
+            || ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.get(settings).equals(List.of(Node.NODE_NAME_SETTING.get(settings)));
     }
 
-    Terminal.Verbosity unexpectedNoopVerbosityLevel() {
-        return Terminal.Verbosity.NORMAL;
-    }
-
-    private static void fullyWriteFile(Path basePath, String fileName, boolean replace,
-                                       CheckedConsumer<OutputStream, Exception> writer) throws Exception {
-        boolean success = false;
+    private static void fullyWriteFile(Path basePath, String fileName, boolean replace, CheckedConsumer<OutputStream, Exception> writer)
+        throws Exception {
         Path filePath = basePath.resolve(fileName);
         if (false == replace && Files.exists(filePath)) {
-            throw new UserException(ExitCodes.IO_ERROR, String.format(Locale.ROOT, "Output file [%s] already exists and " +
-                    "will not be replaced", filePath));
+            throw new UserException(
+                ExitCodes.IO_ERROR,
+                String.format(Locale.ROOT, "Output file [%s] already exists and " + "will not be replaced", filePath)
+            );
         }
-        // the default permission
+        // the default permission, if not replacing; if replacing use the permission of the to be replaced file
         Set<PosixFilePermission> permission = PosixFilePermissions.fromString("rw-rw----");
         // if replacing, use the permission of the replaced file
         if (Files.exists(filePath)) {
@@ -549,26 +642,20 @@ public class ConfigInitialNode extends EnvironmentAwareCommand {
             if (view != null) {
                 view.setPermissions(permission);
             }
-            success = true;
-        } finally {
-            if (success) {
-                if (replace) {
-                    if (Files.exists(filePath, LinkOption.NOFOLLOW_LINKS) &&
-                            false == Files.getOwner(tmpPath, LinkOption.NOFOLLOW_LINKS).equals(Files.getOwner(filePath,
-                                    LinkOption.NOFOLLOW_LINKS))) {
-                        Files.deleteIfExists(tmpPath);
-                        String message = String.format(
-                                Locale.ROOT,
-                                "will not overwrite file at [%s], because this incurs changing the file owner",
-                                filePath);
-                        throw new UserException(ExitCodes.CONFIG, message);
-                    }
-                    Files.move(tmpPath, filePath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
-                } else {
-                    Files.move(tmpPath, filePath, StandardCopyOption.ATOMIC_MOVE);
-                }
+            if (replace) {
+                Files.move(tmpPath, filePath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
+            } else {
+                Files.move(tmpPath, filePath, StandardCopyOption.ATOMIC_MOVE);
             }
+        } finally {
             Files.deleteIfExists(tmpPath);
         }
     }
+
+    private static boolean isDirEmpty(Path path) throws IOException {
+        // Files.list MUST always be used in a try-with-resource construct in order to release the dir file handler
+        try (Stream<Path> dirContentsStream = Files.list(path)) {
+            return false == dirContentsStream.findAny().isPresent();
+        }
+    }
 }

+ 3 - 4
x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/EnrollNodeToCluster.java

@@ -452,7 +452,9 @@ public class EnrollNodeToCluster extends KeyStoreAwareCommand {
                     bw.write(XPackSettings.SECURITY_ENABLED.getKey() + ": true");
                     bw.newLine();
                     bw.newLine();
-                    if (false == env.settings().hasValue(XPackSettings.ENROLLMENT_ENABLED.getKey())) {
+                    // Set enrollment mode to true unless user explicitly set it to false themselves
+                    if (false == (env.settings().hasValue(XPackSettings.ENROLLMENT_ENABLED.getKey())
+                        && false == XPackSettings.ENROLLMENT_ENABLED.get(env.settings()))) {
                         bw.write(XPackSettings.ENROLLMENT_ENABLED.getKey() + ": true");
                         bw.newLine();
                         bw.newLine();
@@ -643,9 +645,6 @@ public class EnrollNodeToCluster extends KeyStoreAwareCommand {
         if (XPackSettings.SECURITY_ENABLED.exists(settings)) {
             throw new UserException(ExitCodes.CONFIG, "Aborting enrolling to cluster. It appears that security is already configured.");
         }
-        if (XPackSettings.ENROLLMENT_ENABLED.exists(settings) && false == XPackSettings.ENROLLMENT_ENABLED.get(settings)) {
-            throw new UserException(ExitCodes.CONFIG, "Aborting enrolling to cluster. Enrollment is explicitly disabled.");
-        }
         if (false == settings.getByPrefix(XPackSettings.TRANSPORT_SSL_PREFIX).isEmpty() ||
             false == settings.getByPrefix(XPackSettings.HTTP_SSL_PREFIX).isEmpty()) {
             throw new UserException(ExitCodes.CONFIG, "Aborting enrolling to cluster. It appears that TLS is already configured.");

+ 0 - 12
x-pack/plugin/security/src/main/bin/elasticsearch-security-config

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License
-# 2.0; you may not use this file except in compliance with the Elastic License
-# 2.0.
-
-ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.ConfigInitialNode \
-  ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
-  ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
-  "$(dirname "$0")/elasticsearch-cli" \
-  -strict "$@"

+ 0 - 21
x-pack/plugin/security/src/main/bin/elasticsearch-security-config.bat

@@ -1,21 +0,0 @@
-@echo off
-
-rem Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-rem or more contributor license agreements. Licensed under the Elastic License
-rem 2.0; you may not use this file except in compliance with the Elastic License
-rem 2.0.
-
-setlocal enabledelayedexpansion
-setlocal enableextensions
-
-set ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.ConfigInitialNode
-set ES_ADDITIONAL_SOURCES=x-pack-env;x-pack-security-env
-set ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli
-call "%~dp0elasticsearch-cli.bat" "-strict" ^
-  %%* ^
-  || goto exit
-
-endlocal
-endlocal
-:exit
-exit /b %ERRORLEVEL%