CLI tool to reconfigure nodes to enroll (#79690)

This change introduces a CLI tool that can be run directly after
installation time in packaged installations, to allow for a node
that was auto-configured to be the initial node of a cluster during
installation ( default installation behavior) to be reconfigured
to join an existing cluster, using an enrollment token.
The use of this tool presumes that the user has the
appropriate permissions to read/write to the installation dirs and
that this node has not been yet started, i.e. this tool is run
directly after installation. It is destructive, as it removes
existing security auto-configuration, and as such it requires an
explicit verification from the user.

This is a follow-up to #7718.

distribution/packages/src/common/scripts/postinst

@@ -79,6 +79,9 @@ if [ "x$IS_UPGRADE" != "xtrue" ]; then
             echo
             echo "The generated password for the elastic built-in superuser is : ${INITIAL_PASSWORD}"
             echo
+            echo "If this node should join an existing cluster, you can reconfigure this with"
+            echo "'/usr/share/elasticsearch/bin/elasticsearch-reconfigure-node --enrollment-token <token-here>'"
+            echo "after creating an enrollment token on your existing cluster."
             echo
             echo "You can complete the following actions at any time:"
             echo

docs/changelog/79690.yaml

@@ -0,0 +1,5 @@
+pr: 79690
+summary: CLI tool to reconfigure nodes to enroll
+area: "Security"
+type: enhancement
+issues: []

qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -234,7 +234,7 @@ public class ArchiveTests extends PackagingTestCase {
             tempDir.resolve("bcprov-jdk15on-1.64.jar")
         );
         Shell.Result result = runElasticsearchStartCommand(null, false, false);
-        assertElasticsearchFailure(result, "java.lang.NoClassDefFoundError: org/bouncycastle/asn1/x509/GeneralName", null);
+        assertElasticsearchFailure(result, "java.lang.NoClassDefFoundError: org/bouncycastle/", null);
         Files.move(
             tempDir.resolve("bcprov-jdk15on-1.64.jar"),
             installation.lib.resolve("tools").resolve("security-cli").resolve("bcprov-jdk15on-1.64.jar")

qa/os/src/test/java/org/elasticsearch/packaging/test/PackagesSecurityAutoConfigurationTests.java

@@ -8,8 +8,10 @@
 
 package org.elasticsearch.packaging.test;
 
+import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.packaging.util.Installation;
 import org.elasticsearch.packaging.util.Packages;
+import org.elasticsearch.packaging.util.Shell;
 import org.junit.BeforeClass;
 
 import java.nio.file.Files;
@@ -19,11 +21,13 @@ import java.util.List;
 import java.util.Optional;
 import java.util.function.Predicate;
 
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
 import static org.elasticsearch.packaging.util.FileUtils.append;
 import static org.elasticsearch.packaging.util.Packages.assertInstalled;
 import static org.elasticsearch.packaging.util.Packages.assertRemoved;
 import static org.elasticsearch.packaging.util.Packages.installPackage;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
@@ -95,6 +99,145 @@ public class PackagesSecurityAutoConfigurationTests extends PackagingTestCase {
         assertThat(configLines, not(hasItem("# have been automatically generated in order to configure Security.               #")));
     }
 
+    public void test50ReconfigureAndEnroll() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        // We cannot run two packaged installations simultaneously here so that we can test that the second node enrolls successfully
+        // We trigger with an invalid enrollment token, to verify that we removed the existing auto-configuration
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("--enrollment-token thisisinvalid", "y", true);
+        assertThat(result.exitCode, equalTo(ExitCodes.DATA_ERROR)); // invalid enrollment token
+        verifySecurityNotAutoConfigured(installation);
+    }
+
+    public void test60ReconfigureWithoutEnrollmentToken() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("", null, true);
+        assertThat(result.exitCode, equalTo(ExitCodes.USAGE)); // missing enrollment token
+        // we fail on command invocation so we don't even try to remove autoconfiguration
+        verifySecurityAutoConfigured(installation);
+    }
+
+    // The following could very well be unit tests but the way we delete files doesn't play well with jimfs
+
+    public void test70ReconfigureFailsWhenTlsAutoConfDirMissing() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+
+        Optional<String> autoConfigDirName = getAutoConfigDirName(installation);
+        // Move instead of delete because Files.deleteIfExists bails on non empty dirs
+        Files.move(installation.config(autoConfigDirName.get()), installation.config("temp-autoconf-dir"));
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("--enrollment-token a-token", "y", true);
+        assertThat(result.exitCode, equalTo(ExitCodes.USAGE)); //
+    }
+
+    public void test71ReconfigureFailsWhenKeyStorePasswordWrong() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        Shell.Result changePassword = installation.executables().keystoreTool.run(
+            "passwd",
+            "some-password" + "\n" + "some-password" + "\n"
+        );
+        assertThat(changePassword.exitCode, equalTo(0));
+        Shell.Result result = installation.executables().nodeReconfigureTool.run(
+            "--enrollment-token a-token",
+            "y" + "\n" + "some-wrong-password",
+            true
+        );
+        assertThat(result.exitCode, equalTo(ExitCodes.IO_ERROR)); //
+        assertThat(result.stderr, containsString("Error was: Provided keystore password was incorrect"));
+    }
+
+    public void test71ReconfigureFailsWhenKeyStoreDoesNotContainExpectedSettings() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        Shell.Result removeSetting = installation.executables().keystoreTool.run(
+            "remove xpack.security.transport.ssl.keystore.secure_password"
+        );
+        assertThat(removeSetting.exitCode, equalTo(0));
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("--enrollment-token a-token", "y", true);
+        assertThat(result.exitCode, equalTo(ExitCodes.IO_ERROR));
+        assertThat(
+            result.stderr,
+            containsString(
+                "elasticsearch.keystore did not contain expected setting [xpack.security.transport.ssl.keystore.secure_password]."
+            )
+        );
+    }
+
+    public void test72ReconfigureFailsWhenConfigurationDoesNotContainSecurityAutoConfig() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        // We remove everything. We don't need to be precise and remove only auto-configuration, the rest are commented out either way
+        Path yml = installation.config("elasticsearch.yml");
+        Files.write(yml, List.of(), TRUNCATE_EXISTING);
+
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("--enrollment-token a-token", "y", true);
+        assertThat(result.exitCode, equalTo(ExitCodes.USAGE)); //
+        assertThat(result.stderr, containsString("Expected configuration is missing from elasticsearch.yml."));
+    }
+
+    public void test72ReconfigureRetainsUserSettings() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        // We remove everything. We don't need to be precise and remove only auto-configuration, the rest are commented out either way
+        Path yml = installation.config("elasticsearch.yml");
+        List<String> allLines = Files.readAllLines(yml);
+        // Replace a comment we know exists in the auto-configuration stanza, with a user defined setting
+        allLines.set(
+            allLines.indexOf("# All the nodes use the same key and certificate on the inter-node connection"),
+            "cluster.name: testclustername"
+        );
+        allLines.add("node.name: testnodename");
+        Files.write(yml, allLines, TRUNCATE_EXISTING);
+
+        // We cannot run two packaged installations simultaneously here so that we can test that the second node enrolls successfully
+        // We trigger with an invalid enrollment token, to verify that we removed the existing auto-configuration
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("--enrollment-token thisisinvalid", "y", true);
+        assertThat(result.exitCode, equalTo(ExitCodes.DATA_ERROR)); // invalid enrollment token
+        verifySecurityNotAutoConfigured(installation);
+        // Check that user configuration , both inside and outside the autocofiguration stanza, was retained
+        Path editedYml = installation.config("elasticsearch.yml");
+        List<String> newConfigurationLines = Files.readAllLines(editedYml);
+        assertThat(newConfigurationLines, hasItem("cluster.name: testclustername"));
+        assertThat(newConfigurationLines, hasItem("node.name: testnodename"));
+    }
+
     private Predicate<String> successfulAutoConfiguration() {
         Predicate<String> p1 = output -> output.contains("Authentication and authorization are enabled.");
         Predicate<String> p2 = output -> output.contains("TLS for the transport and HTTP layers is enabled and configured.");

qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -723,10 +723,12 @@ public abstract class PackagingTestCase extends Assert {
     public static void verifySecurityNotAutoConfigured(Installation es) throws Exception {
         assertThat(getAutoConfigDirName(es).isPresent(), Matchers.is(false));
         if (es.distribution.isPackage()) {
-            assertThat(
-                sh.run(es.executables().keystoreTool + " list").stdout,
-                not(Matchers.containsString("autoconfiguration.password_hash"))
-            );
+            if (Files.exists(es.config("elasticsearch.keystore"))) {
+                assertThat(
+                    sh.run(es.executables().keystoreTool + " list").stdout,
+                    not(Matchers.containsString("autoconfiguration.password_hash"))
+                );
+            }
         }
         List<String> configLines = Files.readAllLines(es.config("elasticsearch.yml"));
         assertThat(configLines, not(contains(containsString("automatically generated in order to configure Security"))));

qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java

@@ -171,6 +171,10 @@ public class Installation {
         }
 
         public Shell.Result run(String args, String input) {
+            return run(args, input, false);
+        }
+
+        public Shell.Result run(String args, String input, boolean ignoreExitCode) {
             String command = path.toString();
             if (Platforms.WINDOWS) {
                 command = "& '" + command + "'";
@@ -184,6 +188,9 @@ public class Installation {
             if (input != null) {
                 command = "echo \"" + input + "\" | " + command;
             }
+            if (ignoreExitCode) {
+                return sh.runIgnoreExitCode(command + " " + args);
+            }
             return sh.run(command + " " + args);
         }
     }
@@ -201,6 +208,7 @@ public class Installation {
         public final Executable setupPasswordsTool = new Executable("elasticsearch-setup-passwords");
         public final Executable resetPasswordTool = new Executable("elasticsearch-reset-password");
         public final Executable createEnrollmentToken = new Executable("elasticsearch-create-enrollment-token");
+        public final Executable nodeReconfigureTool = new Executable("elasticsearch-reconfigure-node");
         public final Executable sqlCli = new Executable("elasticsearch-sql-cli");
         public final Executable syskeygenTool = new Executable("elasticsearch-syskeygen");
         public final Executable usersTool = new Executable("elasticsearch-users");

x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/AutoConfigureNode.java

@@ -39,6 +39,7 @@ import org.elasticsearch.discovery.SettingsBasedSeedHostsProvider;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.CommandLineHttpClient;
 import org.elasticsearch.xpack.core.security.EnrollmentToken;
@@ -70,13 +71,17 @@ import java.security.cert.X509Certificate;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Base64;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.function.BiFunction;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.security.auth.x500.X500Principal;
@@ -120,6 +125,7 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
 
     private final OptionSpec<String> enrollmentTokenParam = parser.accepts("enrollment-token", "The enrollment token to use")
         .withRequiredArg();
+    private final OptionSpec<Void> reconfigure = parser.accepts("reconfigure");
     private final BiFunction<Environment, String, CommandLineHttpClient> clientFunction;
 
     public AutoConfigureNode(BiFunction<Environment, String, CommandLineHttpClient> clientFunction) {
@@ -186,6 +192,13 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
             notifyOfFailure(inEnrollmentMode, terminal, Terminal.Verbosity.NORMAL, ExitCodes.NOOP, msg);
         }
 
+        if (options.has(reconfigure)) {
+            if (false == inEnrollmentMode) {
+                throw new UserException(ExitCodes.USAGE, "enrollment-token is a mandatory parameter.");
+            }
+            env = possibleReconfigureNode(env, terminal);
+        }
+
         // only perform auto-configuration if the existing configuration is not conflicting (eg Security already enabled)
         // if it is, silently skip auto configuration
         checkExistingConfiguration(env.settings(), inEnrollmentMode, terminal);
@@ -239,6 +252,8 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
         final PrivateKey httpKey;
         final X509Certificate httpCert;
         final List<String> transportAddresses;
+        final X500Principal certificatePrincipal = new X500Principal("CN=" + System.getenv("HOSTNAME"));
+        final X500Principal caPrincipal = new X500Principal(AUTO_CONFIG_ALT_DN);
 
         if (inEnrollmentMode) {
             // this is an enrolling node, get HTTP CA key/certificate and transport layer key/certificate from another node
@@ -344,9 +359,6 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
             // this is the initial node, generate HTTP CA key/certificate and transport layer key/certificate ourselves
             try {
                 transportAddresses = List.of();
-                final X500Principal certificatePrincipal = new X500Principal("CN=" + System.getenv("HOSTNAME"));
-                final X500Principal caPrincipal = new X500Principal(AUTO_CONFIG_ALT_DN);
-                final GeneralNames subjectAltNames = getSubjectAltNames();
                 // self-signed CA for transport layer
                 final KeyPair transportCaKeyPair = CertGenUtils.generateKeyPair(TRANSPORT_CA_KEY_SIZE);
                 final PrivateKey transportCaKey = transportCaKeyPair.getPrivate();
@@ -365,7 +377,7 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
                 transportKey = transportKeyPair.getPrivate();
                 transportCert = CertGenUtils.generateSignedCertificate(
                     certificatePrincipal,
-                    subjectAltNames,
+                    getSubjectAltNames(),
                     transportKeyPair,
                     transportCaCert,
                     transportCaKey,
@@ -373,6 +385,7 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
                     TRANSPORT_CERTIFICATE_DAYS,
                     SIGNATURE_ALGORITHM
                 );
+
                 final KeyPair httpCaKeyPair = CertGenUtils.generateKeyPair(HTTP_CA_KEY_SIZE);
                 httpCaKey = httpCaKeyPair.getPrivate();
                 // self-signed CA
@@ -393,18 +406,13 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
                 throw t;
             }
         }
-
-        // in either case, generate this node's HTTP key/certificate
         try {
-            final X500Principal certificatePrincipal = new X500Principal("CN=" + System.getenv("HOSTNAME"));
-            final GeneralNames subjectAltNames = getSubjectAltNames();
-
             final KeyPair httpKeyPair = CertGenUtils.generateKeyPair(HTTP_KEY_SIZE);
             httpKey = httpKeyPair.getPrivate();
             // non-CA
             httpCert = CertGenUtils.generateSignedCertificate(
                 certificatePrincipal,
-                subjectAltNames,
+                getSubjectAltNames(),
                 httpKeyPair,
                 httpCaCert,
                 httpCaKey,
@@ -536,6 +544,8 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
         }
 
         try {
+            // final Environment to be used in the lambda below
+            final Environment localFinalEnv = env;
             List<String> existingConfigLines = Files.readAllLines(ymlPath, StandardCharsets.UTF_8);
             fullyWriteFile(env.configFile(), "elasticsearch.yml", true, stream -> {
                 try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(stream, StandardCharsets.UTF_8))) {
@@ -569,8 +579,8 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
                     bw.newLine();
                     bw.newLine();
                     // Set enrollment mode to true unless user explicitly set it to false themselves
-                    if (false == (env.settings().hasValue(XPackSettings.ENROLLMENT_ENABLED.getKey())
-                        && false == XPackSettings.ENROLLMENT_ENABLED.get(env.settings()))) {
+                    if (false == (localFinalEnv.settings().hasValue(XPackSettings.ENROLLMENT_ENABLED.getKey())
+                        && false == XPackSettings.ENROLLMENT_ENABLED.get(localFinalEnv.settings()))) {
                         bw.write(XPackSettings.ENROLLMENT_ENABLED.getKey() + ": true");
                         bw.newLine();
                         bw.newLine();
@@ -618,8 +628,8 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
                         // we have configured TLS on the transport layer with newly generated certs and keys,
                         // hence this node cannot form a multi-node cluster
                         // if we don't set the following the node might trip the discovery bootstrap check
-                        if (false == DiscoveryModule.isSingleNodeDiscovery(env.settings())
-                            && false == ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.exists(env.settings())) {
+                        if (false == DiscoveryModule.isSingleNodeDiscovery(localFinalEnv.settings())
+                            && false == ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.exists(localFinalEnv.settings())) {
                             bw.newLine();
                             bw.write("# The initial node with security auto-configured must form a cluster on its own,");
                             bw.newLine();
@@ -632,12 +642,12 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
 
                     // if any address settings have been set, assume the admin has thought it through wrt to addresses,
                     // and don't try to be smart and mess with that
-                    if (false == (env.settings().hasValue(HttpTransportSettings.SETTING_HTTP_HOST.getKey())
-                        || env.settings().hasValue(HttpTransportSettings.SETTING_HTTP_BIND_HOST.getKey())
-                        || env.settings().hasValue(HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST.getKey())
-                        || env.settings().hasValue(NetworkService.GLOBAL_NETWORK_HOST_SETTING.getKey())
-                        || env.settings().hasValue(NetworkService.GLOBAL_NETWORK_BIND_HOST_SETTING.getKey())
-                        || env.settings().hasValue(NetworkService.GLOBAL_NETWORK_PUBLISH_HOST_SETTING.getKey()))) {
+                    if (false == (localFinalEnv.settings().hasValue(HttpTransportSettings.SETTING_HTTP_HOST.getKey())
+                        || localFinalEnv.settings().hasValue(HttpTransportSettings.SETTING_HTTP_BIND_HOST.getKey())
+                        || localFinalEnv.settings().hasValue(HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST.getKey())
+                        || localFinalEnv.settings().hasValue(NetworkService.GLOBAL_NETWORK_HOST_SETTING.getKey())
+                        || localFinalEnv.settings().hasValue(NetworkService.GLOBAL_NETWORK_BIND_HOST_SETTING.getKey())
+                        || localFinalEnv.settings().hasValue(NetworkService.GLOBAL_NETWORK_PUBLISH_HOST_SETTING.getKey()))) {
                         bw.newLine();
                         bw.write(
                             "# With security now configured, which includes user authentication over HTTPs, "
@@ -678,6 +688,59 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
         Files.deleteIfExists(keystoreBackupPath);
     }
 
+    private Environment possibleReconfigureNode(Environment env, Terminal terminal) throws UserException {
+        // We remove the existing auto-configuration stanza from elasticsearch.yml, the elastisearch.keystore and
+        // the directory with the auto-configured TLS key material, and then proceed as if elasticsearch is started
+        // with --enrolment-token token, in the first place.
+        final String autoConfigDirName = getAutoConfigDirName(env);
+        final List<String> existingConfigLines;
+        try {
+            existingConfigLines = Files.readAllLines(env.configFile().resolve("elasticsearch.yml"), StandardCharsets.UTF_8);
+        } catch (IOException e) {
+            // This shouldn't happen, we would have failed earlier but we need to catch the exception
+            throw new UserException(ExitCodes.IO_ERROR, "Aborting enrolling to cluster. Unable to read elasticsearch.yml.", e);
+        }
+        final List<String> existingConfigWithoutAutoconfiguration = removePreviousAutoconfiguration(existingConfigLines);
+        if (existingConfigLines.equals(existingConfigWithoutAutoconfiguration) == false) {
+            terminal.println("");
+            terminal.println("This node will be reconfigured to join an existing cluster, using the enrollment token that you provided.");
+            terminal.println("This operation will overwrite the existing configuration. Specifically: ");
+            terminal.println("  - Security auto configuration will be removed from elasticsearch.yml");
+            terminal.println("  - The " + autoConfigDirName + " directory will be removed");
+            terminal.println("  - Security auto configuration related secure settings will be removed from the elasticsearch.keystore");
+            final boolean shouldContinue = terminal.promptYesNo("Do you want to continue with the reconfiguration process", false);
+            if (shouldContinue == false) {
+                throw new UserException(ExitCodes.OK, "User cancelled operation");
+            }
+            removeAutoConfigurationFromKeystore(env, terminal);
+            try {
+                fullyWriteFile(env.configFile(), "elasticsearch.yml", true, stream -> {
+                    try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(stream, StandardCharsets.UTF_8))) {
+                        for (String l : existingConfigWithoutAutoconfiguration) {
+                            bw.write(l);
+                            bw.newLine();
+                        }
+                    }
+                });
+                deleteDirectory(env.configFile().resolve(autoConfigDirName));
+            } catch (Throwable t) {
+                throw new UserException(
+                    ExitCodes.IO_ERROR,
+                    "Aborting enrolling to cluster. Unable to remove existing security configuration.",
+                    t
+                );
+            }
+            // rebuild the environment after removing the settings that were added in auto-configuration.
+            return createEnv(Map.of("path.home", env.settings().get("path.home")));
+        } else {
+            throw new UserException(
+                ExitCodes.USAGE,
+                "Aborting enrolling to cluster. This node doesn't appear to be auto-configured for security. "
+                    + "Expected configuration is missing from elasticsearch.yml."
+            );
+        }
+    }
+
     private void notifyOfFailure(boolean inEnrollmentMode, Terminal terminal, Terminal.Verbosity verbosity, int exitCode, String message)
         throws UserException {
         if (inEnrollmentMode) {
@@ -901,4 +964,136 @@ public class AutoConfigureNode extends EnvironmentAwareCommand {
     private List<String> getTransportAddresses(Map<String, Object> responseMap) {
         return (List<String>) responseMap.get("nodes_addresses");
     }
+
+    private String getAutoConfigDirName(Environment env) throws UserException {
+        final List<String> autoConfigDirNameList;
+        try {
+            autoConfigDirNameList = Files.list(env.configFile())
+                .map(Path::getFileName)
+                .map(Path::toString)
+                .filter(name -> name.startsWith(TLS_CONFIG_DIR_NAME_PREFIX))
+                .collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new UserException(
+                ExitCodes.USAGE,
+                "Aborting enrolling to cluster. Error attempting to find the directory with generated keys and certificates for TLS",
+                e
+            );
+        }
+        if (autoConfigDirNameList.isEmpty()) {
+            throw new UserException(
+                ExitCodes.USAGE,
+                "Aborting enrolling to cluster. This node doesn't appear to be auto-configured for security. "
+                    + "The directory with generated keys and certificates for TLS is missing."
+            );
+        } else if (autoConfigDirNameList.size() > 1) {
+            throw new UserException(
+                ExitCodes.USAGE,
+                "Aborting enrolling to cluster. Multiple directories with generated keys and certificates for TLS found: "
+                    + autoConfigDirNameList
+            );
+        } else {
+            return autoConfigDirNameList.get(0);
+        }
+    }
+
+    /**
+     * Removes existing autoconfiguration from a configuration file, retaining configuration that is user added even within
+     * the auto-configuration stanza. It only matches configuration setting keys for auto-configuration as the values depend
+     * on the timestamp of the installation time so we can't know with certainty that a user has changed the value
+     */
+    static List<String> removePreviousAutoconfiguration(List<String> existingConfigLines) throws UserException {
+        final Pattern pattern = Pattern.compile(
+            "(?s)(" + Pattern.quote(AUTO_CONFIGURATION_START_MARKER) + ".*?" + Pattern.quote(AUTO_CONFIGURATION_END_MARKER) + ")"
+        );
+        final String existingConfigurationAsString = existingConfigLines.stream().collect(Collectors.joining(System.lineSeparator()));
+        final Matcher matcher = pattern.matcher(existingConfigurationAsString);
+        if (matcher.find()) {
+            final String foundAutoConfigurationSettingsAsString = matcher.group(1);
+            final Settings foundAutoConfigurationSettings;
+            try {
+                foundAutoConfigurationSettings = Settings.builder()
+                    .loadFromSource(foundAutoConfigurationSettingsAsString, XContentType.YAML)
+                    .build();
+            } catch (Exception e) {
+                throw new UserException(
+                    ExitCodes.IO_ERROR,
+                    "Aborting enrolling to cluster. Unable to parse existing configuration file. Error was: " + e.getMessage(),
+                    e
+                );
+            }
+            // This is brittle and needs to be updated with every change above
+            final Set<String> expectedAutoConfigurationSettings = new TreeSet<String>(
+                List.of(
+                    "xpack.security.enabled",
+                    "xpack.security.enrollment.enabled",
+                    "xpack.security.transport.ssl.keystore.path",
+                    "xpack.security.transport.ssl.truststore.path",
+                    "xpack.security.transport.ssl.verification_mode",
+                    "xpack.security.http.ssl.enabled",
+                    "xpack.security.transport.ssl.enabled",
+                    "xpack.security.http.ssl.keystore.path",
+                    "cluster.initial_master_nodes",
+                    "http.host"
+                )
+            );
+            final Set<String> userAddedSettings = new HashSet<>(foundAutoConfigurationSettings.keySet());
+            userAddedSettings.removeAll(expectedAutoConfigurationSettings);
+            final List<String> newConfigurationLines = Arrays.stream(
+                existingConfigurationAsString.replace(foundAutoConfigurationSettingsAsString, "").split(System.lineSeparator())
+            ).collect(Collectors.toList());
+            if (false == userAddedSettings.isEmpty()) {
+                for (String key : userAddedSettings) {
+                    newConfigurationLines.add(key + ": " + foundAutoConfigurationSettings.get(key));
+                }
+            }
+            return newConfigurationLines;
+        }
+        return existingConfigLines;
+    }
+
+    private void removeAutoConfigurationFromKeystore(Environment env, Terminal terminal) throws UserException {
+        if (Files.exists(KeyStoreWrapper.keystorePath(env.configFile()))) {
+            try (
+                KeyStoreWrapper existingKeystore = KeyStoreWrapper.load(env.configFile());
+                SecureString keystorePassword = existingKeystore.hasPassword() ? new SecureString(
+                    terminal.readSecret(
+                        "Enter password for the elasticsearch keystore: ",
+                        KeyStoreWrapper.MAX_PASSPHRASE_LENGTH
+                    )
+                ) : new SecureString(new char[0]);
+            ) {
+                existingKeystore.decrypt(keystorePassword.getChars());
+                List<String> secureSettingsToRemove = List.of(
+                    "xpack.security.transport.ssl.keystore.secure_password",
+                    "xpack.security.transport.ssl.truststore.secure_password",
+                    "xpack.security.http.ssl.keystore.secure_password",
+                    "autoconfiguration.password_hash"
+                );
+                for (String setting : secureSettingsToRemove) {
+                    if (existingKeystore.getSettingNames().contains(setting) == false) {
+                        throw new UserException(
+                            ExitCodes.IO_ERROR,
+                            "Aborting enrolling to cluster. Unable to remove existing security configuration, "
+                                + "elasticsearch.keystore did not contain expected setting ["
+                                + setting
+                                + "]."
+                        );
+                    }
+                    existingKeystore.remove(setting);
+                }
+                existingKeystore.save(env.configFile(), keystorePassword.getChars());
+            } catch (Exception e) {
+                terminal.errorPrintln(Terminal.Verbosity.VERBOSE, "");
+                terminal.errorPrintln(Terminal.Verbosity.VERBOSE, ExceptionsHelper.stackTrace(e));
+                terminal.errorPrintln(Terminal.Verbosity.VERBOSE, "");
+                throw new UserException(
+                    ExitCodes.IO_ERROR,
+                    "Aborting enrolling to cluster. Unable to remove existing secure settings. Error was: " + e.getMessage(),
+                    e
+                );
+            }
+        }
+    }
+
 }

x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/AutoConfigureNodeTests.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.cli;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+
+import static org.elasticsearch.xpack.security.cli.AutoConfigureNode.removePreviousAutoconfiguration;
+
+public class AutoConfigureNodeTests extends ESTestCase {
+
+    public void testRemovePreviousAutoconfiguration() throws Exception {
+        final List<String> file1 = List.of(
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "some.setting1: some.value",
+            "some.setting2: some.value",
+            "some.setting3: some.value",
+            "some.setting4: some.value",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line"
+        );
+        final List<String> file2 = List.of(
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "some.setting1: some.value",
+            "some.setting2: some.value",
+            "some.setting3: some.value",
+            "some.setting4: some.value",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            AutoConfigureNode.AUTO_CONFIGURATION_START_MARKER,
+            "cluster.initial_master_nodes: [\"node1\"]",
+            "http.host: [_site_]",
+            "xpack.security.enabled: true",
+            "xpack.security.enrollment.enabled: true",
+            "xpack.security.http.ssl.enabled: true",
+            "xpack.security.http.ssl.keystore.path: /path/to/the/file",
+            "xpack.security.transport.ssl.keystore.path: /path/to/the/file",
+            "xpack.security.transport.ssl.truststore.path: /path/to/the/file",
+            AutoConfigureNode.AUTO_CONFIGURATION_END_MARKER
+        );
+
+        assertEquals(file1, removePreviousAutoconfiguration(file2));
+    }
+
+    public void testRemovePreviousAutoconfigurationRetainsUserAdded() throws Exception {
+        final List<String> file1 = List.of(
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "some.setting1: some.value",
+            "some.setting2: some.value",
+            "some.setting3: some.value",
+            "some.setting4: some.value",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "some.extra.added.setting: value");
+        final List<String> file2 = List.of(
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            "some.setting1: some.value",
+            "some.setting2: some.value",
+            "some.setting3: some.value",
+            "some.setting4: some.value",
+            "# commented out line",
+            "# commented out line",
+            "# commented out line",
+            AutoConfigureNode.AUTO_CONFIGURATION_START_MARKER,
+            "cluster.initial_master_nodes: [\"node1\"]",
+            "http.host: [_site_]",
+            "xpack.security.enabled: true",
+            "xpack.security.enrollment.enabled: true",
+            "xpack.security.http.ssl.enabled: true",
+            "some.extra.added.setting: value",
+            "xpack.security.http.ssl.keystore.path: /path/to/the/file",
+            "xpack.security.transport.ssl.keystore.path: /path/to/the/file",
+            "xpack.security.transport.ssl.truststore.path: /path/to/the/file",
+            "",
+            AutoConfigureNode.AUTO_CONFIGURATION_END_MARKER);
+        assertEquals(file1, removePreviousAutoconfiguration(file2));
+    }
+}

x-pack/plugin/security/src/main/bin/elasticsearch-reconfigure-node

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.
+
+ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.AutoConfigureNode \
+  ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
+  ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
+  "`dirname "$0"`"/elasticsearch-cli \
+  --reconfigure "$@"

x-pack/plugin/security/src/main/bin/elasticsearch-reconfigure-node.bat

@@ -0,0 +1,21 @@
+@echo off
+
+rem Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+rem or more contributor license agreements. Licensed under the Elastic License
+rem 2.0; you may not use this file except in compliance with the Elastic License
+rem 2.0.
+
+setlocal enabledelayedexpansion
+setlocal enableextensions
+
+set ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.AutoConfigureNode
+set ES_ADDITIONAL_SOURCES=x-pack-env;x-pack-security-env
+set ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli
+call "%~dp0elasticsearch-cli.bat" "--reconfigure" ^
+  %%* ^
+  || goto exit
+
+endlocal
+endlocal
+:exit
+exit /b %ERRORLEVEL%