Generate and store password hash for elastic user (#76276)

For package installations (DEB,RPM), we are generating a
random strong password for the elastic user on installation time
so that we can show it to the user.
We subsequently hash and store this password in the
elasticsearch.keystore so that the node can pick it up on the first
run and use it to populate the relevant document for the elastic
user in the security index.
This change implements a class that can be called from the package
installation scripts (postinst) to

- Generate a strong password
- Hash it with the configured(default) password hashing algo
- Store it in the elasticsearch.keystore
- Print it in stdout so that it the bash script can capture it.

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java

@@ -59,6 +59,8 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
     private final ReservedUserInfo bootstrapUserInfo;
     public static final Setting<SecureString> BOOTSTRAP_ELASTIC_PASSWORD = SecureSetting.secureString("bootstrap.password",
             KeyStoreWrapper.SEED_SETTING);
+    public static final Setting<SecureString> AUTOCONFIG_BOOOTSTRAP_ELASTIC_PASSWORD_HASH =
+        SecureSetting.secureString("autoconfig.password_hash", null);
 
     private final NativeUsersStore nativeUsersStore;
     private final AnonymousUser anonymousUser;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/ResetElasticPasswordTool.java

@@ -32,6 +32,7 @@ import java.util.List;
 import java.util.function.Function;
 
 import static org.elasticsearch.xpack.security.tool.CommandLineHttpClient.createURL;
+import static org.elasticsearch.xpack.security.tool.CommandUtils.generatePassword;
 
 public class ResetElasticPasswordTool extends BaseRunAsSuperuserCommand {
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/AutoConfigGenerateElasticPasswordHash.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.enrollment.tool;
+
+import joptsimple.OptionSet;
+
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.KeyStoreAwareCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.authc.support.Hasher;
+
+import static org.elasticsearch.xpack.security.authc.esnative.ReservedRealm.AUTOCONFIG_BOOOTSTRAP_ELASTIC_PASSWORD_HASH;
+import static org.elasticsearch.xpack.security.tool.CommandUtils.generatePassword;
+
+/**
+ * This tool is not meant to be used in user facing CLI tools. It is called by the package installers only upon installation. It
+ * <ul>
+ *     <li>generates a random strong password for the elastic user/li>
+ *     <li>stores a salted hash of that password in the elasticsearch keystore, in the autoconfiguration.password_hash setting</li>
+ * </ul>
+ * This password is subsequently picked up by the node on startup and is set as the password of the elastic user in the security index.
+ *
+ * There is currently no way to set the password of the elasticsearch keystore during package installation. This tool
+ * is called by the package installer only on installation (not on upgrades) so we can be certain that the keystore
+ * has an empty password (obfuscated).
+ *
+ * The generated password is written to stdout upon success. Error messages are printed to stderr.
+ */
+public class AutoConfigGenerateElasticPasswordHash extends KeyStoreAwareCommand {
+
+    public AutoConfigGenerateElasticPasswordHash() {
+        super("Generates a password hash for for the elastic user and stores it in elasticsearch.keystore");
+    }
+
+    public static void main(String[] args) throws Exception {
+        exit(new AutoConfigGenerateElasticPasswordHash().main(args, Terminal.DEFAULT));
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        final Hasher hasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(env.settings()));
+        try (
+            SecureString elasticPassword = new SecureString(generatePassword(20));
+            KeyStoreWrapper nodeKeystore = KeyStoreWrapper.bootstrap(env.configFile(), () -> new SecureString(new char[0]))
+        ) {
+            nodeKeystore.setString(AUTOCONFIG_BOOOTSTRAP_ELASTIC_PASSWORD_HASH.getKey(), hasher.hash(elasticPassword));
+            nodeKeystore.save(env.configFile(), new char[0]);
+            terminal.print(Terminal.Verbosity.NORMAL, elasticPassword.toString());
+        } catch (Exception e) {
+            throw new UserException(ExitCodes.CANT_CREATE, "Failed to generate a password for the elastic user", e);
+        }
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/BootstrapPasswordAndEnrollmentTokenForInitialNode.java

@@ -34,17 +34,16 @@ import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
-import java.security.SecureRandom;
 import java.util.function.Function;
 
 import static org.elasticsearch.xpack.security.tool.CommandLineHttpClient.createURL;
+import static org.elasticsearch.xpack.security.tool.CommandUtils.generatePassword;
 
 public class BootstrapPasswordAndEnrollmentTokenForInitialNode extends KeyStoreAwareCommand {
     private final CheckedFunction<Environment, EnrollmentTokenGenerator, Exception> createEnrollmentTokenFunction;
     private final Function<Environment, CommandLineHttpClient> clientFunction;
     private final CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction;
     private final OptionSpec<Void> includeNodeEnrollmentToken;
-    private final SecureRandom secureRandom = new SecureRandom();
 
     BootstrapPasswordAndEnrollmentTokenForInitialNode() {
         this(
@@ -142,13 +141,4 @@ public class BootstrapPasswordAndEnrollmentTokenForInitialNode extends KeyStoreA
         return createURL(new URL(client.getDefaultURL()), "/_security/user/" + ElasticUser.NAME + "/_password",
             "?pretty");
     }
-
-    protected char[] generatePassword(int passwordLength) {
-        final char[] passwordChars = ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~!@#$%^&*-_=+?").toCharArray();
-        char[] characters = new char[passwordLength];
-        for (int i = 0; i < passwordLength; ++i) {
-            characters[i] = passwordChars[secureRandom.nextInt(passwordChars.length)];
-        }
-        return characters;
-    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommand.java

@@ -32,7 +32,6 @@ import org.elasticsearch.xpack.security.support.FileAttributesChecker;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.nio.file.Path;
-import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -42,6 +41,9 @@ import java.util.Objects;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.security.tool.CommandUtils.generatePassword;
+import static org.elasticsearch.xpack.security.tool.CommandUtils.generateUsername;
+
 /**
  * A {@link KeyStoreAwareCommand} that can be extended fpr any CLI tool that needs to allow a local user with
  * filesystem write access to perform actions on the node as a superuser. It leverages temporary file realm users
@@ -55,7 +57,6 @@ public abstract class BaseRunAsSuperuserCommand extends KeyStoreAwareCommand {
     private final OptionSpecBuilder force;
     private final Function<Environment, CommandLineHttpClient> clientFunction;
     private final CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction;
-    final SecureRandom secureRandom = new SecureRandom();
 
     public BaseRunAsSuperuserCommand(
         Function<Environment, CommandLineHttpClient> clientFunction,
@@ -90,7 +91,7 @@ public abstract class BaseRunAsSuperuserCommand extends KeyStoreAwareCommand {
             settings = env.settings();
         }
 
-        final String username = generateUsername();
+        final String username = generateUsername("autogenerated_", null, 8);
         try (SecureString password = new SecureString(generatePassword(PASSWORD_LENGTH))){
             final Hasher hasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings));
             final Path passwordFile = FileUserPasswdStore.resolveFile(newEnv);
@@ -244,25 +245,6 @@ public abstract class BaseRunAsSuperuserCommand extends KeyStoreAwareCommand {
         }
     }
 
-    protected char[] generatePassword(int passwordLength) {
-        final char[] passwordChars = ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~!@#$%^&*-_=+?").toCharArray();
-        char[] characters = new char[passwordLength];
-        for (int i = 0; i < passwordLength; ++i) {
-            characters[i] = passwordChars[secureRandom.nextInt(passwordChars.length)];
-        }
-        return characters;
-    }
-
-    private String generateUsername() {
-        final char[] usernameChars = ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789").toCharArray();
-        int usernameLength = 8;
-        char[] characters = new char[usernameLength];
-        for (int i = 0; i < usernameLength; ++i) {
-            characters[i] = usernameChars[secureRandom.nextInt(usernameChars.length)];
-        }
-        return "enrollment_autogenerated_" + new String(characters);
-    }
-
     /**
      * This is called after we have created a temporary superuser in the file realm and verified that its
      * credentials work. The username and password of the generated user are passed as parameters. Overriding methods should

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/CommandUtils.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.tool;
+
+import org.elasticsearch.core.Nullable;
+import java.security.SecureRandom;
+
+public class CommandUtils {
+
+    static final SecureRandom SECURE_RANDOM = new SecureRandom();
+
+    /**
+     * Generates a password of a given length from a set of predefined allowed chars.
+     * @param passwordLength the length of the password
+     * @return the char array with the password
+     */
+    public static char[] generatePassword(int passwordLength) {
+        final char[] passwordChars = ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~!@#$%^&*-_=+?").toCharArray();
+        char[] characters = new char[passwordLength];
+        for (int i = 0; i < passwordLength; ++i) {
+            characters[i] = passwordChars[SECURE_RANDOM.nextInt(passwordChars.length)];
+        }
+        return characters;
+    }
+
+    /**
+     * Generates a string that can be used as a username, possibly consisting of a chosen prefix and suffix
+     */
+    protected static String generateUsername(@Nullable String prefix, @Nullable String suffix, int length) {
+        final char[] usernameChars = ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789").toCharArray();
+
+        final String prefixString = null == prefix ? "" : prefix;
+        final String suffixString = null == suffix ? "" : prefix;
+        char[] characters = new char[length];
+        for (int i = 0; i < length; ++i) {
+            characters[i] = usernameChars[SECURE_RANDOM.nextInt(usernameChars.length)];
+        }
+        return prefixString + new String(characters) + suffixString;
+    }
+}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/BootstrapPasswordAndEnrollmentTokenForInitialNodeTests.java

@@ -57,11 +57,6 @@ public class BootstrapPasswordAndEnrollmentTokenForInitialNodeTests extends Comm
         return new BootstrapPasswordAndEnrollmentTokenForInitialNode(environment -> client, environment -> keyStoreWrapper,
             environment -> enrollmentTokenGenerator) {
             @Override
-            protected char[] generatePassword(int passwordLength) {
-                String password = "Aljngvodjb94j8HSY803";
-                return password.toCharArray();
-            }
-            @Override
             protected Environment readSecureSettings(Environment env, SecureString password) {
                 return new Environment(settings, tempDir);
             }
@@ -129,7 +124,7 @@ public class BootstrapPasswordAndEnrollmentTokenForInitialNodeTests extends Comm
         terminal.addSecretInput("password");
         String includeNodeEnrollmentToken = randomBoolean() ? "--include-node-enrollment-token" : "";
         String output = execute(includeNodeEnrollmentToken);
-        assertThat(output, containsString("elastic user password: Aljngvodjb94j8HSY803"));
+        assertThat(output, containsString("elastic user password: "));
         assertThat(output, containsString("CA fingerprint: ce480d53728605674fcfd8ffb51000d8a33bf32de7c7f1e26b4d428" +
             "f8a91362d"));
         assertThat(output, containsString("Kibana enrollment token: eyJ2ZXIiOiI4LjAuMCIsImFkciI6WyJbMTkyLjE2OC4wL" +

x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/AutoConfigGenerateElasticPasswordHashTests.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.enrollment.tool;
+
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.CommandTestCase;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.PathUtilsForTesting;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.authc.support.Hasher;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+import static org.elasticsearch.test.SecurityIntegTestCase.getFastStoredHashAlgoForTests;
+import static org.elasticsearch.xpack.security.authc.esnative.ReservedRealm.AUTOCONFIG_BOOOTSTRAP_ELASTIC_PASSWORD_HASH;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.emptyString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasLength;
+import static org.hamcrest.Matchers.is;
+
+public class AutoConfigGenerateElasticPasswordHashTests extends CommandTestCase {
+
+    private static FileSystem jimfs;
+    private static Hasher hasher;
+    private Path confDir;
+    private Settings settings;
+    private Environment env;
+
+    @BeforeClass
+    public static void muteInFips() {
+        assumeFalse("Can't run in a FIPS JVM, uses keystore that is not password protected", inFipsJvm());
+    }
+
+    @BeforeClass
+    public static void setupJimfs() {
+        Configuration conf = Configuration.unix().toBuilder().setAttributeViews("posix").build();
+        jimfs = Jimfs.newFileSystem(conf);
+        PathUtilsForTesting.installMock(jimfs);
+    }
+
+    @Before
+    public void setup() throws Exception {
+        Path homeDir = jimfs.getPath("eshome");
+        IOUtils.rm(homeDir);
+        confDir = homeDir.resolve("config");
+        Files.createDirectories(confDir);
+        hasher = getFastStoredHashAlgoForTests();
+        settings = Settings.builder()
+            .put("path.home", homeDir)
+            .put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), hasher.name())
+            .build();
+        env = new Environment(AutoConfigGenerateElasticPasswordHashTests.this.settings, confDir);
+        KeyStoreWrapper keystore = KeyStoreWrapper.create();
+        keystore.save(confDir, new char[0]);
+    }
+
+    @AfterClass
+    public static void closeJimfs() throws IOException {
+        if (jimfs != null) {
+            jimfs.close();
+            jimfs = null;
+        }
+    }
+
+    @Override protected Command newCommand() {
+        return new AutoConfigGenerateElasticPasswordHash() {
+            @Override
+            protected Environment createEnv(Map<String, String> settings) throws UserException {
+                return env;
+            }
+        };
+    }
+
+    public void testSuccessfullyGenerateAndStoreHash() throws Exception {
+        execute();
+        assertThat(terminal.getOutput(), hasLength(20));
+        KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.load(env.configFile());
+        assertNotNull(keyStoreWrapper);
+        keyStoreWrapper.decrypt(new char[0]);
+        assertThat(keyStoreWrapper.getSettingNames(),
+            containsInAnyOrder(AUTOCONFIG_BOOOTSTRAP_ELASTIC_PASSWORD_HASH.getKey(), "keystore.seed"));
+    }
+
+    public void testExistingKeystoreWithWrongPassword() throws Exception {
+        KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.load(env.configFile());
+        assertNotNull(keyStoreWrapper);
+        keyStoreWrapper.decrypt(new char[0]);
+        // set a random password so that we fail to decrypt it in GenerateElasticPasswordHash#execute
+        keyStoreWrapper.save(env.configFile(), randomAlphaOfLength(16).toCharArray());
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertThat(e.getMessage(), equalTo("Failed to generate a password for the elastic user"));
+        assertThat(terminal.getOutput(), is(emptyString()));
+    }
+}