[test] Run bin/plugin as right user

Before this commit he tests always run bin/plugin as root which is somewhat
unrealistic and causes trouble (log files owned by root instead of
elasticsearch). After this commit `bin/plugin` runs as root when elasticsearch
is installed via the repository and as elasticsearch otherwise which is much
more realistic.

This also adds extra timeout to starting elasticsearch which is required
when all the plugins are installed. And it fixes up a problem with logging
elasticsearch's log if elasticsearch doesn't start which came up multiple
time while debugging this problem.

Also adds docs recommending running `bin/plugin` as the user that owns the
Elasticsearch files or root if installed with the packages.

Closes #13557

docs/plugins/plugin-script.asciidoc

@@ -13,6 +13,15 @@ Run the following command to get usage instructions:
 sudo bin/plugin -h
 -----------------------------------
 
+[IMPORTANT]
+.Running as root
+=====================
+If Elasticsearch was installed using the deb or rpm package then run
+`bin/plugin` as `root` so it can write to the appropriate files on disk.
+Otherwise run `bin/plugin` as the user that owns all of the Elasticsearch
+files.
+=====================
+
 [[installation]]
 === Installing Plugins
 
@@ -222,4 +231,3 @@ plugin.mandatory: mapper-attachments,lang-groovy
 --------------------------------------------------
 
 For safety reasons, a node will not start if it is missing a mandatory plugin.
-

qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash

@@ -316,15 +316,11 @@ wait_for_elasticsearch_status() {
     echo "Making sure elasticsearch is up..."
     wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200 || {
           echo "Looks like elasticsearch never started. Here is its log:"
-          if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
-              cat /tmp/elasticsearch/logs/elasticsearch.log
+          if [ -e "$ESLOG/elasticsearch.log" ]; then
+              cat "$ESLOG/elasticsearch.log"
           else
-              if [ -e '/var/log/elasticsearch/elasticsearch.log' ]; then
-                  cat /var/log/elasticsearch/elasticsearch.log
-              else
-                  echo "The elasticsearch log doesn't exist. Maybe /vag/log/messages has something:"
-                  tail -n20 /var/log/messages
-              fi
+              echo "The elasticsearch log doesn't exist. Maybe /vag/log/messages has something:"
+              tail -n20 /var/log/messages
           fi
           false
     }
@@ -398,6 +394,7 @@ move_config() {
     mv "$oldConfig"/* "$ESCONFIG"
     chown -R elasticsearch:elasticsearch "$ESCONFIG"
     assert_file_exist "$ESCONFIG/elasticsearch.yml"
+    assert_file_exist "$ESCONFIG/logging.yml"
 }
 
 # Copies a script into the Elasticsearch install.

qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash

@@ -67,6 +67,7 @@ if [[ "$BATS_TEST_FILENAME" =~ 25_tar_plugins.bats$ ]]; then
     }
     export ESHOME=/tmp/elasticsearch
     export_elasticsearch_paths
+    export ESPLUGIN_COMMAND_USER=elasticsearch
 else
     load os_package
     if is_rpm; then
@@ -75,6 +76,7 @@ else
         GROUP='DEB PLUGINS'
     fi
     export_elasticsearch_paths
+    export ESPLUGIN_COMMAND_USER=root
     install() {
         install_package
         verify_package_installation

qa/vagrant/src/test/resources/packaging/scripts/plugins.bash

@@ -32,10 +32,18 @@ install_plugin() {
 
     assert_file_exist "$path"
 
-    "$ESHOME/bin/plugin" install "file://$path"
+    sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/plugin" install "file://$path"
 
     assert_file_exist "$ESPLUGINS/$name"
     assert_file_exist "$ESPLUGINS/$name/plugin-descriptor.properties"
+
+    # At some point installing or removing plugins caused elasticsearch's logs
+    # to be owned by root. This is bad so we want to make sure it doesn't
+    # happen.
+    if [ -e "$ESLOG" ] && [ $(stat "$ESLOG" --format "%U") == "root" ]; then
+        echo "$ESLOG is now owned by root! That'll break logging when elasticsearch tries to start."
+        false
+    fi
 }
 
 install_jvm_plugin() {
@@ -50,9 +58,17 @@ remove_plugin() {
     local name=$1
 
     echo "Removing $name...."
-    "$ESHOME/bin/plugin" remove $name
+    sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/plugin" remove $name
 
     assert_file_not_exist "$ESPLUGINS/$name"
+
+    # At some point installing or removing plugins caused elasticsearch's logs
+    # to be owned by root. This is bad so we want to make sure it doesn't
+    # happen.
+    if [ -e "$ESLOG" ] && [ $(stat "$ESLOG" --format "%U") == "root" ]; then
+        echo "$ESLOG is now owned by root! That'll break logging when elasticsearch tries to start."
+        false
+    fi
 }
 
 # Install the jvm-example plugin which fully excercises the special case file

qa/vagrant/src/test/resources/packaging/scripts/tar.bash

@@ -73,7 +73,6 @@ export_elasticsearch_paths() {
     export ESSCRIPTS="$ESCONFIG/scripts"
     export ESDATA="$ESHOME/data"
     export ESLOG="$ESHOME/logs"
-
 }
 
 # Checks that all directories & files are correctly installed