Migrate certgen packaging test from bats (#50880)

This commit moves the packaging tests for elasticsearch-certgen
to java from bats. Although certgen is deprecated, the tests are
moved rather than just deleted, and the tests themselves should be
easily adaptable to certutil. One note is that the test is simplified to
use a single node, rather than the two node test from bats, which was
problematic given how the newer distro tests only operate with a single
distribution.

relates #46005

qa/os/bats/default/40_tar_certgen.bats

@@ -1 +0,0 @@
-certgen.bash

qa/os/bats/default/45_package_certgen.bats

@@ -1 +0,0 @@
-certgen.bash

qa/os/bats/default/certgen.bash

@@ -1,436 +0,0 @@
-#!/usr/bin/env bats
-
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-
-load $BATS_UTILS/utils.bash
-load $BATS_UTILS/plugins.bash
-load $BATS_UTILS/xpack.bash
-
-# Description of the nodes instances
-instances="/tmp/instances.yml"
-
-# Destination for generated certificates
-certificates="/tmp/certificates.zip"
-
-setup() {
-    if [ $BATS_TEST_NUMBER == 1 ]; then
-        clean_before_test
-    fi
-}
-
-DEFAULT_ARCHIVE_USER=elasticsearch
-DEFAULT_ARCHIVE_ESHOME="/tmp/elasticsearch"
-DEFAULT_ARCHIVE_UTILS=$BATS_UTILS/tar.bash
-
-DEFAULT_PACKAGE_USER=root
-DEFAULT_PACKAGE_ESHOME="/usr/share/elasticsearch"
-DEFAULT_PACKAGE_UTILS=$BATS_UTILS/packages.bash
-
-if [[ "$BATS_TEST_FILENAME" =~ 40_tar_certgen.bats$ ]]; then
-    GROUP='TAR CERTGEN'
-
-    MASTER_USER=$DEFAULT_ARCHIVE_USER
-    MASTER_GROUP=$DEFAULT_ARCHIVE_USER
-    MASTER_DPERMS=755
-    MASTER_HOME=$DEFAULT_ARCHIVE_ESHOME
-    MASTER_UTILS=$DEFAULT_ARCHIVE_UTILS
-
-    DATA_USER=$DEFAULT_PACKAGE_USER
-    DATA_GROUP=elasticsearch
-    DATA_DPERMS=2755
-    DATA_HOME=$DEFAULT_PACKAGE_ESHOME
-    DATA_UTILS=$DEFAULT_PACKAGE_UTILS
-
-    install_master_node() {
-	    install_node_using_archive
-    }
-    start_master_node() {
-	    start_node_using_archive
-    }
-    install_data_node() {
-	    install_node_using_package
-    }
-    start_data_node() {
-	    start_node_using_package
-    }
-else
-    if is_rpm; then
-        GROUP='RPM CERTGEN'
-    elif is_dpkg; then
-        GROUP='DEB CERTGEN'
-    fi
-
-    MASTER_USER=$DEFAULT_PACKAGE_USER
-    MASTER_GROUP=elasticsearch
-    MASTER_DPERMS=2755
-    MASTER_HOME=$DEFAULT_PACKAGE_ESHOME
-    MASTER_UTILS=$DEFAULT_PACKAGE_UTILS
-
-    DATA_USER=$DEFAULT_ARCHIVE_USER
-    DATA_GROUP=$DEFAULT_ARCHIVE_USER
-    DATA_DPERMS=755
-    DATA_HOME=$DEFAULT_ARCHIVE_ESHOME
-    DATA_UTILS=$DEFAULT_ARCHIVE_UTILS
-
-    install_master_node() {
-	    install_node_using_package
-    }
-    start_master_node() {
-	    start_node_using_package
-    }
-    install_data_node() {
-	    install_node_using_archive
-    }
-    start_data_node() {
-	    start_node_using_archive
-    }
-fi
-
-# Install a node with x-pack using the archive file
-install_node_using_archive() {
-    load $BATS_UTILS/tar.bash
-    export ESHOME="$DEFAULT_ARCHIVE_ESHOME"
-    export_elasticsearch_paths
-
-    assert_file_not_exist "/home/elasticsearch"
-    install_archive
-    set_debug_logging
-    verify_archive_installation
-    assert_file_not_exist "/home/elasticsearch"
-
-    export ESPLUGIN_COMMAND_USER=$DEFAULT_ARCHIVE_USER
-    generate_trial_license
-    verify_xpack_installation
-}
-
-# Starts a node installed using the archive
-start_node_using_archive() {
-    load $BATS_UTILS/tar.bash
-    export ESHOME="$DEFAULT_ARCHIVE_ESHOME"
-    export_elasticsearch_paths
-
-    run sudo -u $DEFAULT_ARCHIVE_USER "$ESHOME/bin/elasticsearch" -d -p $ESHOME/elasticsearch.pid
-    [ "$status" -eq "0" ] || {
-	echo "Failed to start node using archive: $output"
-	false
-    }
-}
-
-# Install a node with x-pack using a package file
-install_node_using_package() {
-    load $BATS_UTILS/packages.bash
-    export ESHOME="$DEFAULT_PACKAGE_ESHOME"
-    export_elasticsearch_paths
-
-    assert_file_not_exist "/home/elasticsearch"
-    install_package
-    set_debug_logging
-    verify_package_installation
-
-    export ESPLUGIN_COMMAND_USER=$DEFAULT_PACKAGE_USER
-    generate_trial_license
-    verify_xpack_installation
-}
-
-# Starts a node installed using a package
-start_node_using_package() {
-    if is_systemd; then
-	run systemctl daemon-reload
-        [ "$status" -eq 0 ]
-
-        run sudo systemctl start elasticsearch.service
-        [ "$status" -eq "0" ]
-
-    elif is_sysvinit; then
-        run sudo service elasticsearch start
-        [ "$status" -eq "0" ]
-    fi
-}
-
-
-@test "[$GROUP] install master node" {
-    install_master_node
-}
-
-@test "[$GROUP] add bootstrap password" {
-    load $MASTER_UTILS
-    export ESHOME="$MASTER_HOME"
-    export_elasticsearch_paths
-
-    # For the sake of simplicity we use a bootstrap password in this test. The
-    # alternative would be to start the master node, use
-    # elasticsearch-setup-passwords and restart the node once ssl/tls is
-    # configured. Or use elasticsearch-setup-passwords over HTTPS with the right
-    # cacerts imported into a Java keystore.
-    run sudo -E -u $MASTER_USER bash <<"NEW_PASS"
-if [[ ! -f $ESCONFIG/elasticsearch.keystore ]]; then
-    $ESHOME/bin/elasticsearch-keystore create
-fi
-echo "changeme" | $ESHOME/bin/elasticsearch-keystore add --stdin bootstrap.password
-NEW_PASS
-    [ "$status" -eq 0 ] || {
-        echo "Expected elasticsearch-keystore tool exit code to be zero"
-        echo "$output"
-        false
-    }
-}
-
-@test "[$GROUP] create instances file" {
-    rm -f /tmp/instances.yml
-    run sudo -E -u $MASTER_USER bash <<"CREATE_INSTANCES_FILE"
-cat > /tmp/instances.yml <<- EOF
-instances:
-  - name: "node-master"
-    ip:
-      - "127.0.0.1"
-  - name: "node-data"
-    ip:
-      - "127.0.0.1"
-EOF
-CREATE_INSTANCES_FILE
-
-    [ "$status" -eq 0 ] || {
-        echo "Failed to create instances file [$instances]: $output"
-        false
-    }
-}
-
-@test "[$GROUP] create certificates" {
-    if [[ -f "$certificates" ]]; then
-	    sudo rm -f "$certificates"
-    fi
-
-    run sudo -E -u $MASTER_USER "$MASTER_HOME/bin/elasticsearch-certgen" --in "$instances" --out "$certificates"
-    [ "$status" -eq 0 ] || {
-        echo "Expected elasticsearch-certgen tool exit code to be zero"
-        echo "$output"
-        false
-    }
-
-    echo "$output" | grep "Certificates written to $certificates"
-    assert_file "$certificates" f $MASTER_USER $MASTER_USER 600
-}
-
-@test "[$GROUP] install certificates on master node" {
-    load $MASTER_UTILS
-    export ESHOME="$MASTER_HOME"
-    export_elasticsearch_paths
-
-    certs="$ESCONFIG/certs"
-    if [[ -d "$certs" ]]; then
-	    sudo rm -rf "$certs"
-    fi
-
-    run sudo -E -u $MASTER_USER "unzip" $certificates -d $certs
-    [ "$status" -eq 0 ] || {
-	    echo "Failed to unzip certificates in $certs: $output"
-	    false
-    }
-
-    assert_file "$certs/ca/ca.key" f $MASTER_USER $MASTER_GROUP 644
-    assert_file "$certs/ca/ca.crt" f $MASTER_USER $MASTER_GROUP 644
-
-    assert_file "$certs/node-master" d $MASTER_USER $MASTER_GROUP $MASTER_DPERMS
-    assert_file "$certs/node-master/node-master.key" f $MASTER_USER $MASTER_GROUP 644
-    assert_file "$certs/node-master/node-master.crt" f $MASTER_USER $MASTER_GROUP 644
-
-    assert_file "$certs/node-data" d $MASTER_USER $MASTER_GROUP $MASTER_DPERMS
-    assert_file "$certs/node-data/node-data.key" f $MASTER_USER $MASTER_GROUP 644
-    assert_file "$certs/node-data/node-data.crt" f $MASTER_USER $MASTER_GROUP 644
-}
-
-@test "[$GROUP] update master node settings" {
-    load $MASTER_UTILS
-    export ESHOME="$MASTER_HOME"
-    export_elasticsearch_paths
-
-    run sudo -E -u $MASTER_USER bash <<"MASTER_SETTINGS"
-cat >> $ESCONFIG/elasticsearch.yml <<- EOF
-node.name: "node-master"
-node.master: true
-node.data: false
-discovery.seed_hosts: ["127.0.0.1:9301"]
-cluster.initial_master_nodes: ["node-master"]
-
-xpack.security.transport.ssl.key: $ESCONFIG/certs/node-master/node-master.key
-xpack.security.transport.ssl.certificate: $ESCONFIG/certs/node-master/node-master.crt
-xpack.security.transport.ssl.certificate_authorities: ["$ESCONFIG/certs/ca/ca.crt"]
-xpack.security.http.ssl.key: $ESCONFIG/certs/node-master/node-master.key
-xpack.security.http.ssl.certificate: $ESCONFIG/certs/node-master/node-master.crt
-xpack.security.http.ssl.certificate_authorities: ["$ESCONFIG/certs/ca/ca.crt"]
-
-xpack.security.transport.ssl.enabled: true
-transport.port: 9300
-
-xpack.security.http.ssl.enabled: true
-http.port: 9200
-
-EOF
-MASTER_SETTINGS
-
-    start_master_node
-        wait_for_xpack 127.0.0.1 9200
-}
-
-@test "[$GROUP] test connection to master node using HTTPS" {
-    load $MASTER_UTILS
-    export ESHOME="$MASTER_HOME"
-    export_elasticsearch_paths
-
-    run sudo -E -u $MASTER_USER curl -u "elastic:changeme" --cacert "$ESCONFIG/certs/ca/ca.crt" -XGET "https://127.0.0.1:9200"
-    [ "$status" -eq 0 ] || {
-	    echo "Failed to connect to master node using HTTPS:"
-	    echo "$output"
-	    debug_collect_logs
-	    false
-    }
-    echo "$output" | grep "node-master"
-}
-
-@test "[$GROUP] install data node" {
-    install_data_node
-}
-
-@test "[$GROUP] install certificates on data node" {
-    load $DATA_UTILS
-    export ESHOME="$DATA_HOME"
-    export_elasticsearch_paths
-
-    sudo chown $DATA_USER:$DATA_USER "$certificates"
-    [ -f "$certificates" ] || {
-	    echo "Could not find certificates: $certificates"
-	    false
-    }
-
-    certs="$ESCONFIG/certs"
-    if [[ -d "$certs" ]]; then
-	    sudo rm -rf "$certs"
-    fi
-
-    run sudo -E -u $DATA_USER "unzip" $certificates -d $certs
-    [ "$status" -eq 0 ] || {
-	    echo "Failed to unzip certificates in $certs: $output"
-	    false
-    }
-
-    assert_file "$certs/ca" d $DATA_USER $DATA_GROUP
-    assert_file "$certs/ca/ca.key" f $DATA_USER $DATA_GROUP 644
-    assert_file "$certs/ca/ca.crt" f $DATA_USER $DATA_GROUP 644
-
-    assert_file "$certs/node-master" d $DATA_USER $DATA_GROUP
-    assert_file "$certs/node-master/node-master.key" f $DATA_USER $DATA_GROUP 644
-    assert_file "$certs/node-master/node-master.crt" f $DATA_USER $DATA_GROUP 644
-
-    assert_file "$certs/node-data" d $DATA_USER $DATA_GROUP
-    assert_file "$certs/node-data/node-data.key" f $DATA_USER $DATA_GROUP 644
-    assert_file "$certs/node-data/node-data.crt" f $DATA_USER $DATA_GROUP 644
-}
-
-@test "[$GROUP] update data node settings" {
-    load $DATA_UTILS
-    export ESHOME="$DATA_HOME"
-    export_elasticsearch_paths
-
-    run sudo -E -u $DATA_USER bash <<"DATA_SETTINGS"
-cat >> $ESCONFIG/elasticsearch.yml <<- EOF
-node.name: "node-data"
-node.master: false
-node.data: true
-discovery.seed_hosts: ["127.0.0.1:9300"]
-
-xpack.security.transport.ssl.key: $ESCONFIG/certs/node-data/node-data.key
-xpack.security.transport.ssl.certificate: $ESCONFIG/certs/node-data/node-data.crt
-xpack.security.transport.ssl.certificate_authorities: ["$ESCONFIG/certs/ca/ca.crt"]
-xpack.security.http.ssl.key: $ESCONFIG/certs/node-data/node-data.key
-xpack.security.http.ssl.certificate: $ESCONFIG/certs//node-data/node-data.crt
-xpack.security.http.ssl.certificate_authorities: ["$ESCONFIG/certs/ca/ca.crt"]
-
-xpack.security.transport.ssl.enabled: true
-transport.port: 9301
-
-xpack.security.http.ssl.enabled: true
-http.port: 9201
-
-EOF
-DATA_SETTINGS
-
-    start_data_node
-    wait_for_xpack 127.0.0.1 9201
-}
-
-@test "[$GROUP] test connection to data node using HTTPS" {
-    load $DATA_UTILS
-    export ESHOME="$DATA_HOME"
-    export_elasticsearch_paths
-
-    run sudo -E -u $DATA_USER curl --cacert "$ESCONFIG/certs/ca/ca.crt" -XGET "https://127.0.0.1:9201"
-    [ "$status" -eq 0 ] || {
-	    echo "Failed to connect to data node using HTTPS:"
-	    echo "$output"
-	    false
-    }
-    echo "$output" | grep "missing authentication credentials"
-}
-
-@test "[$GROUP] test node to node communication" {
-    load $MASTER_UTILS
-    export ESHOME="$MASTER_HOME"
-    export_elasticsearch_paths
-
-    testIndex=$(sudo curl -u "elastic:changeme" \
-        -H "Content-Type: application/json" \
-        --cacert "$ESCONFIG/certs/ca/ca.crt" \
-        -XPOST "https://127.0.0.1:9200/books/_doc/0?refresh" \
-        -d '{"title": "Elasticsearch The Definitive Guide"}')
-
-    debug_collect_logs
-    echo "$testIndex" | grep '"result":"created"'
-
-    masterSettings=$(sudo curl -u "elastic:changeme" \
-        -H "Content-Type: application/json" \
-        --cacert "$ESCONFIG/certs/ca/ca.crt" \
-        -XGET "https://127.0.0.1:9200/_nodes/node-master?filter_path=nodes.*.settings.xpack,nodes.*.settings.http.type,nodes.*.settings.transport.type")
-
-    echo "$masterSettings" | grep '"http":{"ssl":{"enabled":"true"}'
-    echo "$masterSettings" | grep '"http":{"type":"security4"}'
-    echo "$masterSettings" | grep '"transport":{"ssl":{"enabled":"true"}'
-    echo "$masterSettings" | grep '"transport":{"type":"security4"}'
-
-    load $DATA_UTILS
-    export ESHOME="$DATA_HOME"
-    export_elasticsearch_paths
-
-    dataSettings=$(curl -u "elastic:changeme" \
-        -H "Content-Type: application/json" \
-        --cacert "$ESCONFIG/certs/ca/ca.crt" \
-        -XGET "https://127.0.0.1:9200/_nodes/node-data?filter_path=nodes.*.settings.xpack,nodes.*.settings.http.type,nodes.*.settings.transport.type")
-
-    echo "$dataSettings" | grep '"http":{"ssl":{"enabled":"true"}'
-    echo "$dataSettings" | grep '"http":{"type":"security4"}'
-    echo "$dataSettings" | grep '"transport":{"ssl":{"enabled":"true"}'
-    echo "$dataSettings" | grep '"transport":{"type":"security4"}'
-
-    testSearch=$(curl -u "elastic:changeme" \
-        -H "Content-Type: application/json" \
-        --cacert "$ESCONFIG/certs/ca/ca.crt" \
-        -XGET "https://127.0.0.1:9200/_search?q=title:guide")
-
-    echo "$testSearch" | grep '"_index":"books"'
-    echo "$testSearch" | grep '"_id":"0"'
-}
-
-@test "[$GROUP] exit code on failure" {
-    run sudo -E -u $MASTER_USER "$MASTER_HOME/bin/elasticsearch-certgen" --not-a-valid-option
-    [ "$status" -ne 0 ] || {
-        echo "Expected elasticsearch-certgen tool exit code to be non-zero"
-        echo "$output"
-        false
-    }
-}
-
-@test "[$GROUP] remove Elasticsearch" {
-    # NOTE: this must be the last test, so that running oss tests does not already have the default distro still installed
-    clean_before_test
-}

qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java

@@ -0,0 +1,119 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.apache.http.client.fluent.Request;
+import org.elasticsearch.packaging.util.Distribution;
+import org.elasticsearch.packaging.util.FileUtils;
+import org.elasticsearch.packaging.util.Platforms;
+import org.elasticsearch.packaging.util.ServerUtils;
+import org.elasticsearch.packaging.util.Shell;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeFalse;
+import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File;
+import static org.elasticsearch.packaging.util.FileMatcher.file;
+import static org.elasticsearch.packaging.util.FileMatcher.p600;
+import static org.elasticsearch.packaging.util.FileUtils.append;
+import static org.elasticsearch.packaging.util.FileUtils.escapePath;
+import static org.elasticsearch.packaging.util.FileUtils.getTempDir;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assume.assumeTrue;
+
+public class CertGenCliTests extends PackagingTestCase {
+    private static final Path instancesFile = getTempDir().resolve("instances.yml");
+    private static final Path certificatesFile = getTempDir().resolve("certificates.zip");
+
+    @Before
+    public void filterDistros() {
+        assumeTrue("only default distro", distribution.flavor == Distribution.Flavor.DEFAULT);
+        assumeTrue("no docker", distribution.packaging != Distribution.Packaging.DOCKER);
+    }
+
+    @BeforeClass
+    public static void cleanupFiles() {
+        FileUtils.rm(instancesFile, certificatesFile);
+    }
+
+    public void test10Install() throws Exception {
+        install();
+    }
+
+    public void test20Help() throws Exception {
+        Shell.Result result = installation.executables().certgenTool.run("--help");
+        assertThat(result.stdout, containsString("Simplifies certificate creation"));
+    }
+
+    public void test30Generate() throws Exception {
+        Files.write(instancesFile, Arrays.asList(
+            "instances:",
+            "  - name: \"mynode\"",
+            "    ip:",
+            "      - \"127.0.0.1\""));
+
+        installation.executables().certgenTool.run("--in " + instancesFile + " --out " + certificatesFile);
+
+        String owner = installation.getOwner();
+        assertThat(certificatesFile, file(File, owner, owner, p600));
+    }
+
+    public void test31ExtractCerts() throws Exception {
+        // windows 2012 r2 has powershell 4.0, which lacks Expand-Archive
+        assumeFalse(Platforms.OS_NAME.equals("Windows Server 2012 R2"));
+
+        Path certsDir = installation.config("certs");
+        sh.extractZip(certificatesFile, certsDir);
+
+        Path caDir = certsDir.resolve("ca");
+        assertThat(caDir.resolve("ca.key"), file(File, null, null, null));
+        assertThat(caDir.resolve("ca.crt"), file(File, null, null, null));
+
+        Path nodeDir = certsDir.resolve("mynode");
+        assertThat(nodeDir.resolve("mynode.key"), file(File, null, null, null));
+        assertThat(nodeDir.resolve("mynode.crt"), file(File, null, null, null));
+
+        FileUtils.cp(certsDir, installation.config("certs"));
+    }
+
+    public void test40RunWithCert() throws Exception {
+        // windows 2012 r2 has powershell 4.0, which lacks Expand-Archive
+        assumeFalse(Platforms.OS_NAME.equals("Windows Server 2012 R2"));
+        
+        append(installation.config("elasticsearch.yml"), String.join("\n",
+            "node.name: mynode",
+            "xpack.security.transport.ssl.key: " + escapePath(installation.config("certs/mynode/mynode.key")),
+            "xpack.security.transport.ssl.certificate: " + escapePath(installation.config("certs/mynode/mynode.crt")),
+            "xpack.security.transport.ssl.certificate_authorities: [\"" + escapePath(installation.config("certs/ca/ca.crt")) + "\"]",
+            "xpack.security.http.ssl.key: " + escapePath(installation.config("certs/mynode/mynode.key")),
+            "xpack.security.http.ssl.certificate: "+ escapePath(installation.config("certs/mynode/mynode.crt")),
+            "xpack.security.http.ssl.certificate_authorities: [\"" + escapePath(installation.config("certs/ca/ca.crt")) + "\"]",
+            "xpack.security.transport.ssl.enabled: true",
+            "xpack.security.http.ssl.enabled: true"));
+
+        assertWhileRunning(() -> {
+            ServerUtils.makeRequest(Request.Get("https://127.0.0.1:9200"), null, null, installation.config("certs/ca/ca.crt"));
+        });
+    }
+}

qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -175,6 +175,12 @@ public abstract class PackagingTestCase extends Assert {
                 logger.warn("Elasticsearch log:\n" +
                     FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"));
             }
+            if (Files.exists(installation.logs.resolve("output.out"))) {
+                logger.warn("Stdout:\n" + FileUtils.slurpTxtorGz(installation.logs.resolve("output.out")));
+            }
+            if (Files.exists(installation.logs.resolve("output.err"))) {
+                logger.warn("Stderr:\n" + FileUtils.slurpTxtorGz(installation.logs.resolve("output.err")));
+            }
             throw e;
         }
 

qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java

@@ -62,7 +62,8 @@ public class PasswordToolsTests extends PackagingTestCase {
             Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
             Map<String, String> userpasses = parseUsersAndPasswords(result.stdout);
             for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
-                String response = ServerUtils.makeRequest(Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue());
+                String response = ServerUtils.makeRequest(
+                    Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue(), null);
                 assertThat(response, containsString("You Know, for Search"));
             }
         });
@@ -111,7 +112,7 @@ public class PasswordToolsTests extends PackagingTestCase {
         assertWhileRunning(() -> {
             String response = ServerUtils.makeRequest(
                 Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"),
-                "elastic", BOOTSTRAP_PASSWORD);
+                "elastic", BOOTSTRAP_PASSWORD, null);
             assertThat(response, containsString("\"status\":\"green\""));
         });
     }
@@ -123,7 +124,8 @@ public class PasswordToolsTests extends PackagingTestCase {
             Map<String, String> userpasses = parseUsersAndPasswords(result.stdout);
             assertThat(userpasses, hasKey("elastic"));
             for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
-                String response = ServerUtils.makeRequest(Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue());
+                String response = ServerUtils.makeRequest(
+                    Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue(), null);
                 assertThat(response, containsString("You Know, for Search"));
             }
         });

qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java

@@ -160,7 +160,7 @@ public class WindowsServiceTests extends PackagingTestCase {
     }
 
     // NOTE: service description is not attainable through any powershell api, so checking it is not possible...
-    public void assertStartedAndStop() throws IOException {
+    public void assertStartedAndStop() throws Exception {
         ServerUtils.waitForElasticsearch(installation);
         ServerUtils.runElasticsearchTests();
 
@@ -191,7 +191,7 @@ public class WindowsServiceTests extends PackagingTestCase {
             "}");
     }
 
-    public void test30StartStop() throws IOException {
+    public void test30StartStop() throws Exception {
         sh.run(serviceScript + " install");
         assertCommand(serviceScript + " start");
         assertStartedAndStop();
@@ -209,7 +209,7 @@ public class WindowsServiceTests extends PackagingTestCase {
         assertThat(result.stdout, containsString("The service '" + DEFAULT_ID + "' has been stopped"));
     }
 
-    public void test33JavaChanged() throws IOException {
+    public void test33JavaChanged() throws Exception {
         final Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
 
         try {

qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java

@@ -26,7 +26,6 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.http.client.fluent.Request;
 import org.elasticsearch.common.CheckedRunnable;
 
-import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.PosixFileAttributes;
@@ -503,7 +502,7 @@ public class Docker {
         }
     }
 
-    public static JsonNode getJson(String path) throws IOException {
+    public static JsonNode getJson(String path) throws Exception {
         final String pluginsResponse = makeRequest(Request.Get("http://localhost:9200/" + path));
 
         ObjectMapper mapper = new ObjectMapper();

qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java

@@ -62,7 +62,7 @@ public class FileMatcher extends TypeSafeMatcher<Path> {
 
     public FileMatcher(Fileness fileness, String owner, String group, Set<PosixFilePermission> posixPermissions) {
         this.fileness = Objects.requireNonNull(fileness);
-        this.owner = Objects.requireNonNull(owner);
+        this.owner = owner;
         this.group = group;
         this.posixPermissions = posixPermissions;
     }
@@ -76,16 +76,18 @@ public class FileMatcher extends TypeSafeMatcher<Path> {
 
         if (Platforms.WINDOWS) {
             final BasicFileAttributes attributes = getBasicFileAttributes(path);
-            final String attributeViewOwner = getFileOwner(path);
 
             if (fileness.equals(Fileness.Directory) != attributes.isDirectory()) {
                 mismatch = "Is " + (attributes.isDirectory() ? "a directory" : "a file");
                 return false;
             }
 
-            if (attributeViewOwner.contains(owner) == false) {
-                mismatch = "Owned by " + attributeViewOwner;
-                return false;
+            if (owner != null) {
+                final String attributeViewOwner = getFileOwner(path);
+                if (attributeViewOwner.contains(owner) == false) {
+                    mismatch = "Owned by " + attributeViewOwner;
+                    return false;
+                }
             }
         } else {
             final PosixFileAttributes attributes = getPosixFileAttributes(path);
@@ -95,7 +97,7 @@ public class FileMatcher extends TypeSafeMatcher<Path> {
                 return false;
             }
 
-            if (owner.equals(attributes.owner().getName()) == false) {
+            if (owner != null && owner.equals(attributes.owner().getName()) == false) {
                 mismatch = "Owned by " + attributes.owner().getName();
                 return false;
             }

qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java

@@ -286,7 +286,7 @@ public class FileUtils {
 
     // vagrant creates /tmp for us in windows so we use that to avoid long paths
     public static Path getTempDir() {
-        return Paths.get("/tmp");
+        return Paths.get("/tmp").toAbsolutePath();
     }
 
     public static Path getDefaultArchiveInstallPath() {
@@ -339,4 +339,15 @@ public class FileUtils {
             }
         }
     }
+
+    /**
+     * Return the given path a string suitable for using on the host system.
+     */
+    public static String escapePath(Path path) {
+        if (Platforms.WINDOWS) {
+            // replace single backslash with forward slash, to avoid unintended escapes in scripts
+            return path.toString().replace('\\', '/');
+        }
+        return path.toString();
+    }
 }

qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java

@@ -114,6 +114,19 @@ public class Installation {
         );
     }
 
+    /**
+     * Returns the user that owns this installation.
+     *
+     * For packages this is root, and for archives it is the user doing the installation.
+     */
+    public String getOwner() {
+        if (Platforms.WINDOWS) {
+            // windows is always administrator, since there is no sudo
+            return "BUILTIN\\Administrators";
+        }
+        return distribution.isArchive() ? ARCHIVE_OWNER : "root";
+    }
+
     public Path bin(String executableName) {
         return bin.resolve(executableName);
     }
@@ -147,7 +160,7 @@ public class Installation {
 
         public Shell.Result run(String args, String input) {
             String command = path + " " + args;
-            if (distribution.isArchive() && distribution.platform != Distribution.Platform.WINDOWS) {
+            if (distribution.isArchive() && Platforms.WINDOWS == false) {
                 command = "sudo -E -u " + ARCHIVE_OWNER + " " + command;
             }
             if (input != null) {
@@ -163,6 +176,7 @@ public class Installation {
         public final Executable pluginTool = new Executable("elasticsearch-plugin");
         public final Executable keystoreTool = new Executable("elasticsearch-keystore");
         public final Executable certutilTool = new Executable("elasticsearch-certutil");
+        public final Executable certgenTool = new Executable("elasticsearch-certgen");
         public final Executable shardTool = new Executable("elasticsearch-shard");
         public final Executable nodeTool = new Executable("elasticsearch-node");
         public final Executable setupPasswordsTool = new Executable("elasticsearch-setup-passwords");

qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java

@@ -305,7 +305,7 @@ public class Packages {
         }
     }
 
-    public static void assertElasticsearchStarted(Shell sh, Installation installation) throws IOException {
+    public static void assertElasticsearchStarted(Shell sh, Installation installation) throws Exception {
         waitForElasticsearch(installation);
 
         if (isSystemd()) {
@@ -324,7 +324,7 @@ public class Packages {
         }
     }
 
-    public static void restartElasticsearch(Shell sh, Installation installation) throws IOException {
+    public static void restartElasticsearch(Shell sh, Installation installation) throws Exception {
         if (isSystemd()) {
             sh.run("systemctl restart elasticsearch.service");
         } else {

qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -23,17 +23,32 @@ import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.fluent.Executor;
 import org.apache.http.client.fluent.Request;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.entity.ContentType;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.util.EntityUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetAddress;
 import java.net.Socket;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.KeyStore;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
@@ -45,24 +60,25 @@ public class ServerUtils {
     private static final Logger logger =  LogManager.getLogger(ServerUtils.class);
 
     private static String SECURITY_ENABLED = "xpack.security.enabled: true";
+    private static String SSL_ENABLED = "xpack.security.http.ssl.enabled: true";
 
     // generous timeout  as nested virtualization can be quite slow ...
     private static final long waitTime = TimeUnit.MINUTES.toMillis(3);
     private static final long timeoutLength = TimeUnit.SECONDS.toMillis(30);
     private static final long requestInterval = TimeUnit.SECONDS.toMillis(5);
 
-    public static void waitForElasticsearch(Installation installation) throws IOException {
-        boolean securityEnabled = false;
+    public static void waitForElasticsearch(Installation installation) throws Exception {
+        boolean xpackEnabled = false;
 
         // TODO: need a way to check if docker has security enabled, the yml config is not bind mounted so can't look from here
         if (installation.distribution.packaging != Distribution.Packaging.DOCKER) {
             Path configFilePath = installation.config("elasticsearch.yml");
             // this is fragile, but currently doesn't deviate from a single line enablement and not worth the parsing effort
             String configFile = Files.readString(configFilePath, StandardCharsets.UTF_8);
-            securityEnabled = configFile.contains(SECURITY_ENABLED);
+            xpackEnabled = configFile.contains(SECURITY_ENABLED) || configFile.contains(SSL_ENABLED);
         }
 
-        if (securityEnabled) {
+        if (xpackEnabled) {
             // with security enabled, we may or may not have setup a user/pass, so we use a more generic port being available check.
             // this isn't as good as a health check, but long term all this waiting should go away when node startup does not
             // make the http port available until the system is really ready to serve requests
@@ -78,17 +94,46 @@ public class ServerUtils {
      * @param request the request to execute
      * @param username the username to supply, or null
      * @param password the password to supply, or null
+     * @param caCert path to the ca certificate the server side ssl cert was generated from, or no if not using ssl
      * @return the response from the server
      * @throws IOException if an error occurs
      */
-    private static HttpResponse execute(Request request, String username, String password) throws IOException {
-        final Executor executor = Executor.newInstance();
+    private static HttpResponse execute(Request request, String username, String password, Path caCert) throws Exception {
+        final Executor executor;
+        if (caCert != null) {
+            try (InputStream inStream = Files.newInputStream(caCert)) {
+                CertificateFactory cf = CertificateFactory.getInstance("X.509");
+                X509Certificate cert = (X509Certificate) cf.generateCertificate(inStream);
+                KeyStore truststore = KeyStore.getInstance(KeyStore.getDefaultType());
+                truststore.load(null, null);
+                truststore.setCertificateEntry("myClusterCA", cert);
+                KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+                kmf.init(truststore, null);
+                TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+                tmf.init(truststore);
+                SSLContext context = SSLContext.getInstance("TLSv1.2");
+                context.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
+                final LayeredConnectionSocketFactory ssl = new SSLConnectionSocketFactory(context);
+                final Registry<ConnectionSocketFactory> sfr = RegistryBuilder.<ConnectionSocketFactory>create()
+                    .register("http", PlainConnectionSocketFactory.getSocketFactory())
+                    .register("https", ssl)
+                    .build();
+                PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(sfr);
+                connectionManager.setDefaultMaxPerRoute(100);
+                connectionManager.setMaxTotal(200);
+                connectionManager.setValidateAfterInactivity(1000);
+                executor = Executor.newInstance(HttpClientBuilder.create()
+                    .setConnectionManager(connectionManager)
+                    .build());
+            }
+        } else {
+            executor = Executor.newInstance();
+        }
 
         if (username != null && password != null) {
             executor.auth(username, password);
             executor.authPreemptive(new HttpHost("localhost", 9200));
         }
-
         return executor.execute(request).returnResponse();
     }
 
@@ -118,7 +163,7 @@ public class ServerUtils {
         Installation installation,
         String username,
         String password
-    ) throws IOException {
+    ) throws Exception {
 
         Objects.requireNonNull(status);
 
@@ -129,6 +174,11 @@ public class ServerUtils {
         boolean started = false;
         Throwable thrownException = null;
 
+        Path caCert = installation.config("certs/ca/ca.crt");
+        if (Files.exists(caCert) == false) {
+            caCert = null; // no cert, so don't use ssl
+        }
+
         while (started == false && timeElapsed < waitTime) {
             if (System.currentTimeMillis() - lastRequest > requestInterval) {
                 try {
@@ -139,7 +189,8 @@ public class ServerUtils {
                             .connectTimeout((int) timeoutLength)
                             .socketTimeout((int) timeoutLength),
                         username,
-                        password
+                        password,
+                        caCert
                     );
 
                     if (response.getStatusLine().getStatusCode() >= 300) {
@@ -179,11 +230,11 @@ public class ServerUtils {
             url = "http://localhost:9200/_cluster/health/" + index + "?wait_for_status=" + status + "&timeout=60s&pretty";
         }
 
-        final String body = makeRequest(Request.Get(url), username, password);
+        final String body = makeRequest(Request.Get(url), username, password, caCert);
         assertThat("cluster health response must contain desired status", body, containsString(status));
     }
 
-    public static void runElasticsearchTests() throws IOException {
+    public static void runElasticsearchTests() throws Exception {
         makeRequest(
             Request.Post("http://localhost:9200/library/_doc/1?refresh=true&pretty")
                 .bodyString("{ \"title\": \"Book #1\", \"pages\": 123 }", ContentType.APPLICATION_JSON));
@@ -198,12 +249,12 @@ public class ServerUtils {
         makeRequest(Request.Delete("http://localhost:9200/_all"));
     }
 
-    public static String makeRequest(Request request) throws IOException {
-        return makeRequest(request, null, null);
+    public static String makeRequest(Request request) throws Exception {
+        return makeRequest(request, null, null, null);
     }
 
-    public static String makeRequest(Request request, String username, String password) throws IOException {
-        final HttpResponse response = execute(request, username, password);
+    public static String makeRequest(Request request, String username, String password, Path caCert) throws Exception {
+        final HttpResponse response = execute(request, username, password, caCert);
         final String body = EntityUtils.toString(response.getEntity());
 
         if (response.getStatusLine().getStatusCode() >= 300) {

qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java

@@ -98,7 +98,12 @@ public class Shell {
         ));
     }
 
-    public Result run( String command, Object... args) {
+    public void extractZip(Path zipPath, Path destinationDir) throws Exception {
+        Platforms.onLinux(() -> run("unzip \"" + zipPath + "\" -d \"" + destinationDir + "\""));
+        Platforms.onWindows(() -> run("Expand-Archive -Path \"" + zipPath + "\" -DestinationPath \"" + destinationDir + "\""));
+    }
+
+    public Result run(String command, Object... args) {
         String formattedCommand = String.format(Locale.ROOT, command, args);
         return run(formattedCommand);
     }