Upgrade keystore on package install (#41755)

When Elasticsearch is run from a package installation, the running
process does not have permissions to write to the keystore. This is
because of the root:root ownership of /etc/elasticsearch. This is why we
create the keystore if it does not exist during package installation. If
the keystore needs to be upgraded, that is currently done by the running
Elasticsearch process. Yet, as just mentioned, the Elasticsearch process
would not have permissions to do that during runtime. Instead, this
needs to be done during package upgrade. This commit adds an upgrade
command to the keystore CLI for this purpose, and that is invoked during
package upgrade if the keystore already exists. This ensures that we are
always on the latest keystore format before the Elasticsearch process is
invoked, and therefore no upgrade would be needed then. While this bug
has always existed, we have not heard of reports of it in practice. Yet,
this bug becomes a lot more likely with a recent change to the format of
the keystore to remove the distinction between file and string entries.

distribution/packages/src/common/scripts/postinst

@@ -94,11 +94,15 @@ elif [ "$RESTART_ON_UPGRADE" = "true" ]; then
 fi
 
 # the equivalent code for rpm is in posttrans
-if [ "$PACKAGE" = "deb" -a ! -f /etc/elasticsearch/elasticsearch.keystore ]; then
-    /usr/share/elasticsearch/bin/elasticsearch-keystore create
-    chown root:elasticsearch /etc/elasticsearch/elasticsearch.keystore
-    chmod 660 /etc/elasticsearch/elasticsearch.keystore
-    md5sum /etc/elasticsearch/elasticsearch.keystore > /etc/elasticsearch/.elasticsearch.keystore.initial_md5sum
+if [ "$PACKAGE" = "deb" ]; then
+    if [ ! -f /etc/elasticsearch/elasticsearch.keystore ]; then
+        /usr/share/elasticsearch/bin/elasticsearch-keystore create
+        chown root:elasticsearch /etc/elasticsearch/elasticsearch.keystore
+        chmod 660 /etc/elasticsearch/elasticsearch.keystore
+        md5sum /etc/elasticsearch/elasticsearch.keystore > /etc/elasticsearch/.elasticsearch.keystore.initial_md5sum
+    else
+        /usr/share/elasticsearch/bin/elasticsearch-keystore upgrade
+    fi
 fi
 
 ${scripts.footer}

distribution/packages/src/common/scripts/posttrans

@@ -3,6 +3,8 @@ if [ ! -f /etc/elasticsearch/elasticsearch.keystore ]; then
     chown root:elasticsearch /etc/elasticsearch/elasticsearch.keystore
     chmod 660 /etc/elasticsearch/elasticsearch.keystore
     md5sum /etc/elasticsearch/elasticsearch.keystore > /etc/elasticsearch/.elasticsearch.keystore.initial_md5sum
+else
+    /usr/share/elasticsearch/bin/elasticsearch-keystore upgrade
 fi
 
 ${scripts.footer}

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java

@@ -34,6 +34,7 @@ public class KeyStoreCli extends LoggingAwareMultiCommand {
         subcommands.put("add", new AddStringKeyStoreCommand());
         subcommands.put("add-file", new AddFileKeyStoreCommand());
         subcommands.put("remove", new RemoveSettingKeyStoreCommand());
+        subcommands.put("upgrade", new UpgradeKeyStoreCommand());
     }
 
     public static void main(String[] args) throws Exception {

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/UpgradeKeyStoreCommand.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A sub-command for the keystore CLI that enables upgrading the keystore format.
+ */
+public class UpgradeKeyStoreCommand extends EnvironmentAwareCommand {
+
+    UpgradeKeyStoreCommand() {
+        super("Upgrade the keystore format");
+    }
+
+    @Override
+    protected void execute(final Terminal terminal, final OptionSet options, final Environment env) throws Exception {
+        final KeyStoreWrapper wrapper = KeyStoreWrapper.load(env.configFile());
+        if (wrapper == null) {
+            throw new UserException(
+                    ExitCodes.CONFIG,
+                    "keystore does not exist at [" + KeyStoreWrapper.keystorePath(env.configFile()) + "]");
+        }
+        wrapper.decrypt(new char[0]);
+        KeyStoreWrapper.upgrade(wrapper, env.configFile(), new char[0]);
+    }
+
+}

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/UpgradeKeyStoreCommandTests.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasToString;
+
+public class UpgradeKeyStoreCommandTests extends KeyStoreCommandTestCase {
+
+    @Override
+    protected Command newCommand() {
+        return new UpgradeKeyStoreCommand() {
+
+            @Override
+            protected Environment createEnv(final Map<String, String> settings) {
+                return env;
+            }
+
+        };
+    }
+
+    public void testKeystoreUpgrade() throws Exception {
+        final Path keystore = KeyStoreWrapper.keystorePath(env.configFile());
+        try (InputStream is = KeyStoreWrapperTests.class.getResourceAsStream("/format-v3-elasticsearch.keystore");
+             OutputStream os = Files.newOutputStream(keystore)) {
+            is.transferTo(os);
+        }
+        try (KeyStoreWrapper beforeUpgrade = KeyStoreWrapper.load(env.configFile())) {
+            assertNotNull(beforeUpgrade);
+            assertThat(beforeUpgrade.getFormatVersion(), equalTo(3));
+        }
+        execute();
+        try (KeyStoreWrapper afterUpgrade = KeyStoreWrapper.load(env.configFile())) {
+            assertNotNull(afterUpgrade);
+            assertThat(afterUpgrade.getFormatVersion(), equalTo(KeyStoreWrapper.FORMAT_VERSION));
+            afterUpgrade.decrypt(new char[0]);
+            assertThat(afterUpgrade.getSettingNames(), hasItem(KeyStoreWrapper.SEED_SETTING.getKey()));
+        }
+    }
+
+    public void testKeystoreDoesNotExist() {
+        final UserException e = expectThrows(UserException.class, this::execute);
+        assertThat(e, hasToString(containsString("keystore does not exist at [" + KeyStoreWrapper.keystorePath(env.configFile()) + "]")));
+    }
+
+}

qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats

@@ -60,6 +60,13 @@ setup() {
     install_package -v $(cat upgrade_from_version)
 }
 
+@test "[UPGRADE] modify keystore" {
+    # deliberately modify the keystore to force it to be preserved during package upgrade
+    export_elasticsearch_paths
+    sudo -E "$ESHOME/bin/elasticsearch-keystore" remove keystore.seed
+    sudo -E echo keystore_seed | "$ESHOME/bin/elasticsearch-keystore" add -x keystore.seed
+}
+
 @test "[UPGRADE] start old version" {
     export JAVA_HOME=$SYSTEM_JAVA_HOME
     start_elasticsearch_service

server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java

@@ -100,7 +100,7 @@ public class KeyStoreWrapper implements SecureSettings {
     private static final String KEYSTORE_FILENAME = "elasticsearch.keystore";
 
     /** The version of the metadata written before the keystore data. */
-    private static final int FORMAT_VERSION = 4;
+    static final int FORMAT_VERSION = 4;
 
     /** The oldest metadata format version that can be read. */
     private static final int MIN_FORMAT_VERSION = 1;