Packaging: Use of default CONF_DIR/CONF_FILE in plugin install

The bin/plugin script now uses the default CONF_DIR & CONF_FILE environment vars. This allows to install a plugin even if Elasticsearch has been installed with a RPM or a DEB package. This commit also adds testing files for TAR archive and plugins installation.

Closes #10673

bin/plugin

@@ -21,6 +21,43 @@ ES_HOME=`dirname "$SCRIPT"`/..
 # make ELASTICSEARCH_HOME absolute
 ES_HOME=`cd "$ES_HOME"; pwd`
 
+# Sets the default values for elasticsearch variables used in this script
+if [ -z "$CONF_DIR" ]; then
+  CONF_DIR="${packaging.plugin.default.config.dir}"
+
+  if [ -z "$CONF_FILE" ]; then
+    CONF_FILE="$CONF_DIR/elasticsearch.yml"
+  fi
+fi
+
+if [ -z "$CONF_FILE" ]; then
+  CONF_FILE="${packaging.plugin.default.config.file}"
+fi
+
+# The default env file is defined at building/packaging time.
+# For a ${packaging.type} package, the value is "${packaging.env.file}".
+ES_ENV_FILE="${packaging.env.file}"
+
+# If an include is specified with the ES_INCLUDE environment variable, use it
+if [ -n "$ES_INCLUDE" ]; then
+    ES_ENV_FILE="$ES_INCLUDE"
+fi
+
+# Source the environment file
+if [ -n "$ES_ENV_FILE" ]; then
+
+  # If the ES_ENV_FILE is not found, try to resolve the path
+  # against the ES_HOME directory
+  if [ ! -f "$ES_ENV_FILE" ]; then
+      ES_ENV_FILE="$ELASTIC_HOME/$ES_ENV_FILE"
+  fi
+
+  . "$ES_ENV_FILE"
+  if [ $? -ne 0 ]; then
+      echo "Unable to source environment file: $ES_ENV_FILE" >&2
+      exit 1
+  fi
+fi
 
 if [ -x "$JAVA_HOME/bin/java" ]; then
     JAVA=$JAVA_HOME/bin/java
@@ -45,5 +82,25 @@ while [ $# -gt 0 ]; do
   shift
 done
 
-exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Xmx64m -Xms16m -Delasticsearch -Des.path.home="$ES_HOME" $properties -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginManager $args
+# check if properties already has a config file or config dir
+if [ -e "$CONF_DIR" ]; then
+  case "$properties" in
+    *-Des.default.path.conf=*|*-Des.path.conf=*)
+    ;;
+    *)
+      properties="$properties -Des.default.path.conf=$CONF_DIR"
+    ;;
+  esac
+fi
 
+if [ -e "$CONF_FILE" ]; then
+  case "$properties" in
+    *-Des.default.config=*|*-Des.config=*)
+    ;;
+    *)
+      properties="$properties -Des.default.config=$CONF_FILE"
+    ;;
+  esac
+fi
+
+exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Xmx64m -Xms16m -Delasticsearch -Des.path.home="$ES_HOME" $properties -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginManager $args

src/packaging/common/packaging.properties

@@ -6,6 +6,10 @@
 # Environment file
 packaging.env.file=
 
+# Default configuration directory and file to use in bin/plugin script
+packaging.plugin.default.config.dir=$ES_HOME/config
+packaging.plugin.default.config.file=$ES_HOME/config/elasticsearch.yml
+
 # Default values for min/max heap memory allocated to elasticsearch java process
 packaging.elasticsearch.heap.min=256m
 packaging.elasticsearch.heap.max=1g
@@ -17,7 +21,7 @@ packaging.os.max.open.files=65535
 packaging.os.max.map.count=262144
 
 # Simple marker to check that properties are correctly overridden
-packaging.type=tar.gz,gzip
+packaging.type=tar.gz
 
 # Custom header for package scripts
 packaging.scripts.header=

src/packaging/deb/packaging.properties

@@ -4,6 +4,10 @@
 # Environment file
 packaging.env.file=/etc/default/elasticsearch
 
+# Default configuration directory and file to use in bin/plugin script
+packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir}
+packaging.plugin.default.config.file=${packaging.elasticsearch.conf.dir}/elasticsearch.yml
+
 # Simple marker to check that properties are correctly overridden
 packaging.type=deb
 

src/packaging/rpm/packaging.properties

@@ -4,6 +4,10 @@
 # Environment file
 packaging.env.file=/etc/sysconfig/elasticsearch
 
+# Default configuration directory and file to use in bin/plugin script
+packaging.plugin.default.config.dir=${packaging.elasticsearch.conf.dir}
+packaging.plugin.default.config.file=${packaging.elasticsearch.conf.dir}/elasticsearch.yml
+
 # Simple marker to check that properties are correctly overridden
 packaging.type=rpm
 

src/test/resources/packaging/scripts/20_tar_package.bats

@@ -0,0 +1,95 @@
+#!/usr/bin/env bats
+
+# This file is used to test the tar gz package.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It removes the 'elasticsearch'
+# user/group and also many directories. Do not execute this file
+# unless you know exactly what you are doing.
+
+# The test case can be executed with the Bash Automated
+# Testing System tool available at https://github.com/sstephenson/bats
+# Thanks to Sam Stephenson!
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Load test utilities
+load packaging_test_utils
+
+# Cleans everything for the 1st execution
+setup() {
+    if [ "$BATS_TEST_NUMBER" -eq 1 ]; then
+        clean_before_test
+    fi
+}
+
+##################################
+# Install TAR GZ package
+##################################
+@test "[TAR] tar command is available" {
+    skip_not_tar_gz
+    run tar --version
+    [ "$status" -eq 0 ]
+}
+
+@test "[TAR] archive is available" {
+    skip_not_tar_gz
+    count=$(find . -type f -name 'elasticsearch*.tar.gz' | wc -l)
+    [ "$count" -eq 1 ]
+}
+
+@test "[TAR] archive is not installed" {
+    skip_not_tar_gz
+    count=$(find /tmp -type d -name 'elasticsearch*' | wc -l)
+    [ "$count" -eq 0 ]
+}
+
+@test "[TAR] install archive" {
+    skip_not_tar_gz
+
+    # Install the archive
+    install_archive
+
+    count=$(find /tmp -type d -name 'elasticsearch*' | wc -l)
+    [ "$count" -eq 1 ]
+}
+
+##################################
+# Check that the archive is correctly installed
+##################################
+@test "[TAR] verify archive installation" {
+    skip_not_tar_gz
+
+    verify_archive_installation "/tmp/elasticsearch"
+}
+
+##################################
+# Check that Elasticsearch is working
+##################################
+@test "[TAR] test elasticsearch" {
+    skip_not_tar_gz
+
+    start_elasticsearch_service
+
+    run_elasticsearch_tests
+
+    stop_elasticsearch_service
+
+    run rm -rf "/tmp/elasticsearch"
+    [ "$status" -eq 0 ]
+}

src/test/resources/packaging/scripts/25_tar_plugins.bats

@@ -0,0 +1,344 @@
+#!/usr/bin/env bats
+
+# This file is used to test the installation and removal
+# of plugins with a tar gz archive.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It removes the 'elasticsearch'
+# user/group and also many directories. Do not execute this file
+# unless you know exactly what you are doing.
+
+# The test case can be executed with the Bash Automated
+# Testing System tool available at https://github.com/sstephenson/bats
+# Thanks to Sam Stephenson!
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Load test utilities
+load packaging_test_utils
+
+setup() {
+    # Cleans everything for every test execution
+    clean_before_test
+
+    # Download Marvel and Shield
+    MARVEL_ZIP="$PWD/marvel.zip"
+    SHIELD_ZIP="$PWD/shield.zip"
+
+    if [ "$BATS_TEST_NUMBER" -eq 1 ]; then
+        if [ ! -e "$MARVEL_ZIP" ]; then
+            wget --quiet -O "$MARVEL_ZIP" "http://download.elasticsearch.org/elasticsearch/marvel/marvel-latest.zip"
+        fi
+        if [ ! -e "$SHIELD_ZIP" ]; then
+            wget --quiet -O "$SHIELD_ZIP" "http://download.elasticsearch.org/elasticsearch/shield/shield-latest.zip"
+        fi
+    fi
+}
+
+##################################
+# Install plugins with a tar archive
+##################################
+@test "[TAR] install marvel plugin" {
+
+    # Install the archive
+    install_archive
+
+    # Checks that the archive is correctly installed
+    verify_archive_installation
+
+    # Checks that plugin archive is available
+    [ -e "$MARVEL_ZIP" ]
+
+    # Install Marvel
+    run /tmp/elasticsearch/bin/plugin -i elasticsearch/marvel/latest -u "file://$MARVEL_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Marvel is correctly installed
+    assert_file_exist "/tmp/elasticsearch/plugins/marvel"
+
+    start_elasticsearch_service
+
+    run curl -XGET 'http://localhost:9200/_cat/plugins?v=false&h=component'
+    [ "$status" -eq 0 ]
+    echo "$output" | grep -w "marvel"
+
+    stop_elasticsearch_service
+
+    # Remove the plugin
+    run /tmp/elasticsearch/bin/plugin -r elasticsearch/marvel/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "/tmp/elasticsearch/plugins/marvel"
+}
+
+@test "[TAR] install marvel plugin with a custom path.plugins" {
+
+    # Install the archive
+    install_archive
+
+    # Checks that the archive is correctly installed
+    verify_archive_installation
+
+    # Creates a temporary directory
+    TEMP_PLUGINS_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'tmp'`
+
+    # Modify the path.plugins setting in configuration file
+    echo "path.plugins: $TEMP_PLUGINS_DIR" >> "/tmp/elasticsearch/config/elasticsearch.yml"
+
+    run chown -R elasticsearch:elasticsearch "$TEMP_PLUGINS_DIR"
+    [ "$status" -eq 0 ]
+
+    # Checks that plugin archive is available
+    [ -e "$MARVEL_ZIP" ]
+
+    # Install Marvel
+    run /tmp/elasticsearch/bin/plugin -i elasticsearch/marvel/latest -u "file://$MARVEL_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Marvel is correctly installed
+    assert_file_exist "$TEMP_PLUGINS_DIR/marvel"
+
+    start_elasticsearch_service
+
+    run curl -XGET 'http://localhost:9200/_cat/plugins?v=false&h=component'
+    [ "$status" -eq 0 ]
+    echo "$output" | grep -w "marvel"
+
+    stop_elasticsearch_service
+
+    # Remove the plugin
+    run /tmp/elasticsearch/bin/plugin -r elasticsearch/marvel/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "$TEMP_PLUGINS_DIR/marvel"
+
+    # Delete the custom plugins directory
+    run rm -rf "$TEMP_PLUGINS_DIR"
+    [ "$status" -eq 0 ]
+}
+
+@test "[TAR] install shield plugin" {
+
+    # Install the archive
+    install_archive
+
+    # Checks that the archive is correctly installed
+    verify_archive_installation
+
+    # Checks that plugin archive is available
+    [ -e "$SHIELD_ZIP" ]
+
+    # Install Shield
+    run /tmp/elasticsearch/bin/plugin -i elasticsearch/shield/latest -u "file://$SHIELD_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Shield is correctly installed
+    assert_file_exist "/tmp/elasticsearch/bin/shield"
+    assert_file_exist "/tmp/elasticsearch/bin/shield/esusers"
+    assert_file_exist "/tmp/elasticsearch/bin/shield/syskeygen"
+    assert_file_exist "/tmp/elasticsearch/config/shield"
+    assert_file_exist "/tmp/elasticsearch/config/shield/role_mapping.yml"
+    assert_file_exist "/tmp/elasticsearch/config/shield/roles.yml"
+    assert_file_exist "/tmp/elasticsearch/config/shield/users"
+    assert_file_exist "/tmp/elasticsearch/config/shield/users_roles"
+    assert_file_exist "/tmp/elasticsearch/plugins/shield"
+
+    # Remove the plugin
+    run /tmp/elasticsearch/bin/plugin -r elasticsearch/shield/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "/tmp/elasticsearch/bin/shield"
+    assert_file_exist "/tmp/elasticsearch/config/shield"
+    assert_file_exist "/tmp/elasticsearch/config/shield/role_mapping.yml"
+    assert_file_exist "/tmp/elasticsearch/config/shield/roles.yml"
+    assert_file_exist "/tmp/elasticsearch/config/shield/users"
+    assert_file_exist "/tmp/elasticsearch/config/shield/users_roles"
+    assert_file_not_exist "/tmp/elasticsearch/plugins/shield"
+}
+
+@test "[TAR] install shield plugin with a custom path.plugins" {
+
+    # Install the archive
+    install_archive
+
+    # Checks that the archive is correctly installed
+    verify_archive_installation
+
+    # Creates a temporary directory
+    TEMP_PLUGINS_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'tmp'`
+
+    # Modify the path.plugins setting in configuration file
+    echo "path.plugins: $TEMP_PLUGINS_DIR" >> "/tmp/elasticsearch/config/elasticsearch.yml"
+
+    run chown -R elasticsearch:elasticsearch "$TEMP_PLUGINS_DIR"
+    [ "$status" -eq 0 ]
+
+    # Checks that plugin archive is available
+    [ -e "$SHIELD_ZIP" ]
+
+    # Install Shield
+    run /tmp/elasticsearch/bin/plugin -i elasticsearch/shield/latest -u "file://$SHIELD_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Shield is correctly installed
+    assert_file_exist "/tmp/elasticsearch/bin/shield"
+    assert_file_exist "/tmp/elasticsearch/bin/shield/esusers"
+    assert_file_exist "/tmp/elasticsearch/bin/shield/syskeygen"
+    assert_file_exist "/tmp/elasticsearch/config/shield"
+    assert_file_exist "/tmp/elasticsearch/config/shield/role_mapping.yml"
+    assert_file_exist "/tmp/elasticsearch/config/shield/roles.yml"
+    assert_file_exist "/tmp/elasticsearch/config/shield/users"
+    assert_file_exist "/tmp/elasticsearch/config/shield/users_roles"
+    assert_file_exist "$TEMP_PLUGINS_DIR/shield"
+
+    # Remove the plugin
+    run /tmp/elasticsearch/bin/plugin -r elasticsearch/shield/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "/tmp/elasticsearch/bin/shield"
+    assert_file_exist "/tmp/elasticsearch/config/shield"
+    assert_file_exist "/tmp/elasticsearch/config/shield/role_mapping.yml"
+    assert_file_exist "/tmp/elasticsearch/config/shield/roles.yml"
+    assert_file_exist "/tmp/elasticsearch/config/shield/users"
+    assert_file_exist "/tmp/elasticsearch/config/shield/users_roles"
+    assert_file_not_exist "$TEMP_PLUGINS_DIR/shield"
+
+    # Delete the custom plugins directory
+    run rm -rf "$TEMP_PLUGINS_DIR"
+    [ "$status" -eq 0 ]
+}
+
+@test "[TAR] install shield plugin with a custom CONFIG_DIR" {
+
+    # Install the archive
+    install_archive
+
+    # Checks that the archive is correctly installed
+    verify_archive_installation
+
+    # Creates a temporary directory
+    TEMP_CONFIG_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'tmp'`
+
+    # Move configuration files to the new configuration directory
+    run mv /tmp/elasticsearch/config/* $TEMP_CONFIG_DIR
+    [ "$status" -eq 0 ]
+
+    run chown -R elasticsearch:elasticsearch "$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+
+    assert_file_exist "$TEMP_CONFIG_DIR/elasticsearch.yml"
+
+    # Checks that plugin archive is available
+    [ -e "$SHIELD_ZIP" ]
+
+    # Install Shield with the CONF_DIR environment variable
+    run env "CONF_DIR=$TEMP_CONFIG_DIR" /tmp/elasticsearch/bin/plugin -i "elasticsearch/shield/latest" -u "file://$SHIELD_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Shield is correctly installed
+    assert_file_exist "/tmp/elasticsearch/bin/shield"
+    assert_file_exist "/tmp/elasticsearch/bin/shield/esusers"
+    assert_file_exist "/tmp/elasticsearch/bin/shield/syskeygen"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/role_mapping.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/roles.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users_roles"
+    assert_file_exist "/tmp/elasticsearch/plugins/shield"
+
+    # Remove the plugin
+    run /tmp/elasticsearch/bin/plugin -r elasticsearch/shield/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "/tmp/elasticsearch/bin/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/role_mapping.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/roles.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users_roles"
+    assert_file_not_exist "/tmp/elasticsearch/plugins/shield"
+
+    # Delete the custom plugins directory
+    run rm -rf "$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+}
+
+@test "[TAR] install shield plugin with a custom ES_JAVA_OPTS" {
+
+    # Install the archive
+    install_archive
+
+    # Checks that the archive is correctly installed
+    verify_archive_installation
+
+    # Creates a temporary directory
+    TEMP_CONFIG_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'tmp'`
+
+    # Move configuration files to the new configuration directory
+    run mv /tmp/elasticsearch/config/* $TEMP_CONFIG_DIR
+    [ "$status" -eq 0 ]
+
+    run chown -R elasticsearch:elasticsearch "$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+
+    assert_file_exist "$TEMP_CONFIG_DIR/elasticsearch.yml"
+
+    # Export ES_JAVA_OPTS
+    export ES_JAVA_OPTS="-Des.path.conf=$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+
+    # Checks that plugin archive is available
+    [ -e "$SHIELD_ZIP" ]
+
+    # Install Shield
+    run /tmp/elasticsearch/bin/plugin -i elasticsearch/shield/latest -u "file://$SHIELD_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Shield is correctly installed
+    assert_file_exist "/tmp/elasticsearch/bin/shield"
+    assert_file_exist "/tmp/elasticsearch/bin/shield/esusers"
+    assert_file_exist "/tmp/elasticsearch/bin/shield/syskeygen"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/role_mapping.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/roles.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users_roles"
+    assert_file_exist "/tmp/elasticsearch/plugins/shield"
+
+    # Remove the plugin
+    run /tmp/elasticsearch/bin/plugin -r elasticsearch/shield/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "/tmp/elasticsearch/bin/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/role_mapping.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/roles.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users_roles"
+    assert_file_not_exist "/tmp/elasticsearch/plugins/shield"
+
+    # Delete the custom plugins directory
+    run rm -rf "$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+}

src/test/resources/packaging/scripts/30_deb_package.bats

@@ -84,7 +84,7 @@ setup() {
 ##################################
 # Check that Elasticsearch is working
 ##################################
-@test "[TEST] test elasticsearch" {
+@test "[DEB] test elasticsearch" {
     skip_not_dpkg
 
     start_elasticsearch_service
@@ -118,6 +118,7 @@ setup() {
     # The removal must disable the service
     # see prerm file
     if is_systemd; then
+        # Debian systemd distros usually returns exit code 3
         run systemctl status elasticsearch.service
         [ "$status" -eq 3 ]
 

src/test/resources/packaging/scripts/40_rpm_package.bats

@@ -83,7 +83,7 @@ setup() {
 ##################################
 # Check that Elasticsearch is working
 ##################################
-@test "[TEST] test elasticsearch" {
+@test "[RPM] test elasticsearch" {
     skip_not_rpm
 
     start_elasticsearch_service
@@ -116,8 +116,10 @@ setup() {
     # The removal must disable the service
     # see prerm file
     if is_systemd; then
+        # Redhat based systemd distros usually returns exit code 1
+        # OpenSUSE13 returns 0
         run systemctl status elasticsearch.service
-        echo "$output" | grep "Active:" | grep 'inactive\|failed'
+        [ "$status" -eq 1 ] || [ "$status" -eq 0 ]
 
         run systemctl is-enabled elasticsearch.service
         [ "$status" -eq 1 ]

src/test/resources/packaging/scripts/50_plugins.bats

@@ -0,0 +1,380 @@
+#!/usr/bin/env bats
+
+# This file is used to test the installation and removal
+# of plugins when Elasticsearch is installed as a DEB/RPM
+# package.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It removes the 'elasticsearch'
+# user/group and also many directories. Do not execute this file
+# unless you know exactly what you are doing.
+
+# The test case can be executed with the Bash Automated
+# Testing System tool available at https://github.com/sstephenson/bats
+# Thanks to Sam Stephenson!
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Load test utilities
+load packaging_test_utils
+
+setup() {
+    # Cleans everything for every test execution
+    clean_before_test
+
+    # Download Marvel and Shield
+    MARVEL_ZIP="$PWD/marvel.zip"
+    SHIELD_ZIP="$PWD/shield.zip"
+
+    if [ "$BATS_TEST_NUMBER" -eq 1 ]; then
+        if [ ! -e "$MARVEL_ZIP" ]; then
+            wget --quiet -O "$MARVEL_ZIP" "http://download.elasticsearch.org/elasticsearch/marvel/marvel-latest.zip"
+        fi
+        if [ ! -e "$SHIELD_ZIP" ]; then
+            wget --quiet -O "$SHIELD_ZIP" "http://download.elasticsearch.org/elasticsearch/shield/shield-latest.zip"
+        fi
+    fi
+}
+
+# Install a deb or rpm package
+install_package() {
+    if is_rpm; then
+        run rpm -i elasticsearch*.rpm >&2
+        [ "$status" -eq 0 ]
+
+    elif is_dpkg; then
+        run dpkg -i elasticsearch*.deb >&2
+        [ "$status" -eq 0 ]
+    fi
+}
+
+##################################
+# Install plugins with DEB/RPM package
+##################################
+@test "[PLUGINS] install marvel plugin" {
+
+    # Install the package
+    install_package
+
+    # Checks that the package is correctly installed
+    verify_package_installation
+
+    # Checks that plugin archive is available
+    [ -e "$MARVEL_ZIP" ]
+
+    # Install Marvel
+    run /usr/share/elasticsearch/bin/plugin -i elasticsearch/marvel/latest -u "file://$MARVEL_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Marvel is correctly installed
+    assert_file_exist "/usr/share/elasticsearch/plugins/marvel"
+
+    start_elasticsearch_service
+
+    run curl -XGET 'http://localhost:9200/_cat/plugins?v=false&h=component'
+    [ "$status" -eq 0 ]
+    echo "$output" | grep -w "marvel"
+
+    stop_elasticsearch_service
+
+    # Remove the plugin
+    run /usr/share/elasticsearch/bin/plugin -r elasticsearch/marvel/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "/usr/share/elasticsearch/plugins/marvel"
+}
+
+@test "[PLUGINS] install marvel plugin with a custom path.plugins" {
+
+    # Install the package
+    install_package
+
+    # Checks that the package is correctly installed
+    verify_package_installation
+
+    # Creates a temporary directory
+    TEMP_PLUGINS_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'tmp'`
+
+    # Modify the path.plugins setting in configuration file
+    echo "path.plugins: $TEMP_PLUGINS_DIR" >> "/etc/elasticsearch/elasticsearch.yml"
+
+    # Sets privileges
+    run chown -R root:elasticsearch "$TEMP_PLUGINS_DIR"
+    [ "$status" -eq 0 ]
+
+    run chmod -R 750 "$TEMP_PLUGINS_DIR"
+    [ "$status" -eq 0 ]
+
+    # Checks that plugin archive is available
+    [ -e "$MARVEL_ZIP" ]
+
+    # Install Marvel
+    run /usr/share/elasticsearch/bin/plugin -i elasticsearch/marvel/latest -u "file://$MARVEL_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Marvel is correctly installed
+    assert_file_exist "$TEMP_PLUGINS_DIR/marvel"
+
+    start_elasticsearch_service
+
+    run curl -XGET 'http://localhost:9200/_cat/plugins?v=false&h=component'
+    [ "$status" -eq 0 ]
+    echo "$output" | grep -w "marvel"
+
+    stop_elasticsearch_service
+
+    # Remove the plugin
+    run /usr/share/elasticsearch/bin/plugin -r elasticsearch/marvel/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "$TEMP_PLUGINS_DIR/marvel"
+
+    # Delete the custom plugins directory
+    run rm -rf "$TEMP_PLUGINS_DIR"
+    [ "$status" -eq 0 ]
+}
+
+@test "[PLUGINS] install shield plugin" {
+
+    # Install the package
+    install_package
+
+    # Checks that the package is correctly installed
+    verify_package_installation
+
+    # Checks that plugin archive is available
+    [ -e "$SHIELD_ZIP" ]
+
+    # Install Shield
+    run /usr/share/elasticsearch/bin/plugin -i elasticsearch/shield/latest -u "file://$SHIELD_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Shield is correctly installed
+    assert_file_exist "/usr/share/elasticsearch/bin/shield"
+    assert_file_exist "/usr/share/elasticsearch/bin/shield/esusers"
+    assert_file_exist "/usr/share/elasticsearch/bin/shield/syskeygen"
+    assert_file_exist "/etc/elasticsearch/shield"
+    assert_file_exist "/etc/elasticsearch/shield/role_mapping.yml"
+    assert_file_exist "/etc/elasticsearch/shield/roles.yml"
+    assert_file_exist "/etc/elasticsearch/shield/users"
+    assert_file_exist "/etc/elasticsearch/shield/users_roles"
+    assert_file_exist "/usr/share/elasticsearch/plugins/shield"
+
+    # Remove the plugin
+    run /usr/share/elasticsearch/bin/plugin -r elasticsearch/shield/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "/usr/share/elasticsearch/bin/shield"
+    assert_file_exist "/etc/elasticsearch/shield"
+    assert_file_exist "/etc/elasticsearch/shield/role_mapping.yml"
+    assert_file_exist "/etc/elasticsearch/shield/roles.yml"
+    assert_file_exist "/etc/elasticsearch/shield/users"
+    assert_file_exist "/etc/elasticsearch/shield/users_roles"
+    assert_file_not_exist "/usr/share/elasticsearch/plugins/shield"
+}
+
+@test "[PLUGINS] install shield plugin with a custom path.plugins" {
+
+    # Install the package
+    install_package
+
+    # Checks that the package is correctly installed
+    verify_package_installation
+
+    # Creates a temporary directory
+    TEMP_PLUGINS_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'tmp'`
+
+    # Modify the path.plugins setting in configuration file
+    echo "path.plugins: $TEMP_PLUGINS_DIR" >> "/etc/elasticsearch/elasticsearch.yml"
+
+    # Sets privileges
+    run chown -R root:elasticsearch "$TEMP_PLUGINS_DIR"
+    [ "$status" -eq 0 ]
+
+    run chmod -R 750 "$TEMP_PLUGINS_DIR"
+    [ "$status" -eq 0 ]
+
+    # Checks that plugin archive is available
+    [ -e "$SHIELD_ZIP" ]
+
+    # Install Shield
+    run /usr/share/elasticsearch/bin/plugin -i elasticsearch/shield/latest -u "file://$SHIELD_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Shield is correctly installed
+    assert_file_exist "/usr/share/elasticsearch/bin/shield"
+    assert_file_exist "/usr/share/elasticsearch/bin/shield/esusers"
+    assert_file_exist "/usr/share/elasticsearch/bin/shield/syskeygen"
+    assert_file_exist "/etc/elasticsearch/shield"
+    assert_file_exist "/etc/elasticsearch/shield/role_mapping.yml"
+    assert_file_exist "/etc/elasticsearch/shield/roles.yml"
+    assert_file_exist "/etc/elasticsearch/shield/users"
+    assert_file_exist "/etc/elasticsearch/shield/users_roles"
+    assert_file_exist "$TEMP_PLUGINS_DIR/shield"
+
+    # Remove the plugin
+    run /usr/share/elasticsearch/bin/plugin -r elasticsearch/shield/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "/usr/share/elasticsearch/bin/shield"
+    assert_file_exist "/etc/elasticsearch/shield"
+    assert_file_exist "/etc/elasticsearch/shield/role_mapping.yml"
+    assert_file_exist "/etc/elasticsearch/shield/roles.yml"
+    assert_file_exist "/etc/elasticsearch/shield/users"
+    assert_file_exist "/etc/elasticsearch/shield/users_roles"
+    assert_file_not_exist "$TEMP_PLUGINS_DIR/marvel"
+
+    # Delete the custom plugins directory
+    run rm -rf "$TEMP_PLUGINS_DIR"
+    [ "$status" -eq 0 ]
+}
+
+@test "[PLUGINS] install shield plugin with a custom CONFIG_DIR" {
+
+    # Install the package
+    install_package
+
+    # Checks that the package is correctly installed
+    verify_package_installation
+
+    # Creates a temporary directory
+    TEMP_CONFIG_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'tmp'`
+
+    # Modify the CONF_DIR variable in environment file
+    if is_rpm; then
+        echo "CONF_DIR=$TEMP_CONFIG_DIR" >> "/etc/sysconfig/elasticsearch"
+    elif is_dpkg; then
+        echo "CONF_DIR=$TEMP_CONFIG_DIR" >> "/etc/default/elasticsearch"
+    fi
+
+    # Move configuration files to the new configuration directory
+    run mv /etc/elasticsearch/* $TEMP_CONFIG_DIR
+    [ "$status" -eq 0 ]
+
+    assert_file_exist "$TEMP_CONFIG_DIR/elasticsearch.yml"
+
+    # Sets privileges
+    run chown -R root:elasticsearch "$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+
+    run chmod -R 750 "$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+
+    # Checks that plugin archive is available
+    [ -e "$SHIELD_ZIP" ]
+
+    # Install Shield
+    run /usr/share/elasticsearch/bin/plugin -i elasticsearch/shield/latest -u "file://$SHIELD_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Shield is correctly installed
+    assert_file_exist "/usr/share/elasticsearch/bin/shield"
+    assert_file_exist "/usr/share/elasticsearch/bin/shield/esusers"
+    assert_file_exist "/usr/share/elasticsearch/bin/shield/syskeygen"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/role_mapping.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/roles.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users"
+    assert_file_exist "/$TEMP_CONFIG_DIR/shield/users_roles"
+    assert_file_exist "/usr/share/elasticsearch/plugins/shield"
+
+    # Remove the plugin
+    run /usr/share/elasticsearch/bin/plugin -r elasticsearch/shield/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "/usr/share/elasticsearch/bin/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/role_mapping.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/roles.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users_roles"
+    assert_file_not_exist "/tmp/elasticsearch/plugins/shield"
+
+    # Delete the custom plugins directory
+    run rm -rf "$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+}
+
+@test "[PLUGINS] install shield plugin with a custom ES_JAVA_OPTS" {
+
+    # Install the package
+    install_package
+
+    # Checks that the package is correctly installed
+    verify_package_installation
+
+    # Creates a temporary directory
+    TEMP_CONFIG_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'tmp'`
+
+    # Move configuration files to the new configuration directory
+    run mv /etc/elasticsearch/* $TEMP_CONFIG_DIR
+    [ "$status" -eq 0 ]
+
+    assert_file_exist "$TEMP_CONFIG_DIR/elasticsearch.yml"
+
+    # Sets privileges
+    run chown -R root:elasticsearch "$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+
+    run chmod -R 750 "$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+
+    # Export ES_JAVA_OPTS
+    export ES_JAVA_OPTS="-Des.path.conf=$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+
+    # Checks that plugin archive is available
+    [ -e "$SHIELD_ZIP" ]
+
+    # Install Shield
+    run /usr/share/elasticsearch/bin/plugin -i elasticsearch/shield/latest -u "file://$SHIELD_ZIP"
+    [ "$status" -eq 0 ]
+
+    # Checks that Shield is correctly installed
+    assert_file_exist "/usr/share/elasticsearch/bin/shield"
+    assert_file_exist "/usr/share/elasticsearch/bin/shield/esusers"
+    assert_file_exist "/usr/share/elasticsearch/bin/shield/syskeygen"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/role_mapping.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/roles.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users_roles"
+    assert_file_exist "/usr/share/elasticsearch/plugins/shield"
+
+    # Remove the plugin
+    run /usr/share/elasticsearch/bin/plugin -r elasticsearch/shield/latest
+    [ "$status" -eq 0 ]
+
+    # Checks that the plugin is correctly removed
+    assert_file_not_exist "/usr/share/elasticsearch/bin/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/role_mapping.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/roles.yml"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users"
+    assert_file_exist "$TEMP_CONFIG_DIR/shield/users_roles"
+    assert_file_not_exist "/usr/share/elasticsearch/plugins/shield"
+
+    # Delete the custom plugins directory
+    run rm -rf "$TEMP_CONFIG_DIR"
+    [ "$status" -eq 0 ]
+}

src/test/resources/packaging/scripts/packaging_test_utils.bash

@@ -171,6 +171,7 @@ assert_output() {
 }
 
 # Checks that all directories & files are correctly installed
+# after a package (deb/rpm) install
 verify_package_installation() {
 
     run id elasticsearch
@@ -218,6 +219,68 @@ verify_package_installation() {
     fi
 }
 
+
+# Install the tar.gz archive
+install_archive() {
+    local eshome="/tmp"
+    if [ "x$1" != "x" ]; then
+        eshome="$1"
+    fi
+
+    run tar -xzvf elasticsearch*.tar.gz -C "$eshome" >&2
+    [ "$status" -eq 0 ]
+
+    run find "$eshome" -depth -type d -name 'elasticsearch*' -exec mv {} "$eshome/elasticsearch" \;
+    [ "$status" -eq 0 ]
+
+    # ES cannot run as root so create elasticsearch user & group if needed
+    if ! getent group "elasticsearch" > /dev/null 2>&1 ; then
+        if is_dpkg; then
+            run addgroup --system "elasticsearch"
+            [ "$status" -eq 0 ]
+        else
+            run groupadd -r "elasticsearch"
+            [ "$status" -eq 0 ]
+        fi
+    fi
+    if ! id "elasticsearch" > /dev/null 2>&1 ; then
+        if is_dpkg; then
+            run adduser --quiet --system --no-create-home --ingroup "elasticsearch" --disabled-password --shell /bin/false "elasticsearch"
+            [ "$status" -eq 0 ]
+        else
+            run useradd --system -M --gid "elasticsearch" --shell /sbin/nologin --comment "elasticsearch user" "elasticsearch"
+            [ "$status" -eq 0 ]
+        fi
+    fi
+
+    run chown -R elasticsearch:elasticsearch "$eshome/elasticsearch"
+    [ "$status" -eq 0 ]
+}
+
+
+# Checks that all directories & files are correctly installed
+# after a archive (tar.gz/zip) install
+verify_archive_installation() {
+    local eshome="/tmp/elasticsearch"
+    if [ "x$1" != "x" ]; then
+        eshome="$1"
+    fi
+
+    assert_file "$eshome" d
+    assert_file "$eshome/bin" d
+    assert_file "$eshome/bin/elasticsearch" f
+    assert_file "$eshome/bin/elasticsearch.in.sh" f
+    assert_file "$eshome/bin/plugin" f
+    assert_file "$eshome/config" d
+    assert_file "$eshome/config/elasticsearch.yml" f
+    assert_file "$eshome/config/logging.yml" f
+    assert_file "$eshome/config" d
+    assert_file "$eshome/lib" d
+    assert_file "$eshome/NOTICE.txt" f
+    assert_file "$eshome/LICENSE.txt" f
+    assert_file "$eshome/README.textile" f
+}
+
 # Deletes everything before running a test file
 clean_before_test() {
 
@@ -230,7 +293,8 @@ clean_before_test() {
                             "/etc/default/elasticsearch" \
                             "/etc/sysconfig/elasticsearch"  \
                             "/var/run/elasticsearch"  \
-                            "/usr/share/doc/elasticsearch")
+                            "/usr/share/doc/elasticsearch" \
+                            "/tmp/elasticsearch")
 
     if [ "$ES_CLEAN_BEFORE_TEST" = "true" ]; then
         # Kills all processes of user elasticsearch
@@ -238,13 +302,16 @@ clean_before_test() {
             pkill -u elasticsearch 2>/dev/null || true
         fi
 
+        # Kills all running Elasticsearch processes
+        ps aux | grep -i "org.elasticsearch.bootstrap.Elasticsearch" | awk {'print $2'} | xargs kill -9 > /dev/null 2>&1 || true
+
         # Removes RPM package
         if is_rpm; then
-            rpm --quiet -e elasticsearch 2>/dev/null || true
+            rpm --quiet -e elasticsearch > /dev/null 2>&1 || true
         fi
 
         if [ -x "`which yum 2>/dev/null`" ]; then
-            yum remove -y elasticsearch 2>/dev/null || true
+            yum remove -y elasticsearch > /dev/null 2>&1 || true
         fi
 
         # Removes DEB package
@@ -280,7 +347,11 @@ clean_before_test() {
 
 start_elasticsearch_service() {
 
-    if is_systemd; then
+    if [ -f "/tmp/elasticsearch/bin/elasticsearch" ]; then
+        run /bin/su -s /bin/sh -c '/tmp/elasticsearch/bin/elasticsearch -d -p /tmp/elasticsearch/elasticsearch.pid' elasticsearch
+        [ "$status" -eq 0 ]
+
+    elif is_systemd; then
         run systemctl daemon-reload
         [ "$status" -eq 0 ]
 
@@ -300,7 +371,14 @@ start_elasticsearch_service() {
 
     wait_for_elasticsearch_status
 
-    if is_systemd; then
+    if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
+        pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
+        [ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
+
+        run  ps $pid
+        [ "$status" -eq 0 ]
+
+    elif is_systemd; then
         run systemctl is-active elasticsearch.service
         [ "$status" -eq 0 ]
 
@@ -315,13 +393,21 @@ start_elasticsearch_service() {
 
 stop_elasticsearch_service() {
 
-    if is_systemd; then
+    if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
+        pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
+        [ "x$pid" != "x" ] && [ "$pid" -gt 0 ]
+
+        run kill -SIGTERM $pid
+        [ "$status" -eq 0 ]
+
+    elif is_systemd; then
         run systemctl stop elasticsearch.service
         [ "$status" -eq 0 ]
 
         run systemctl is-active elasticsearch.service
         [ "$status" -eq 3 ]
-        [ "$output" = "inactive" ]
+
+        echo "$output" | grep -E 'inactive|failed'
 
     elif is_sysvinit; then
         run service elasticsearch stop
@@ -340,8 +426,8 @@ wait_for_elasticsearch_status() {
     fi
 
     # Try to connect to elasticsearch and wait for expected status
-    wget --quiet --retry-connrefused --waitretry=1 --timeout=20 \
-         --output-document=/dev/null "http://localhost:9200/_cluster/health?wait_for_status=$status&timeout=20s"
+    wget --quiet --retry-connrefused --waitretry=1 --timeout=60 \
+         --output-document=/dev/null "http://localhost:9200/_cluster/health?wait_for_status=$status&timeout=60s" || true
 
     # Checks the cluster health
     curl -XGET 'http://localhost:9200/_cat/health?h=status&v=false'