Improve distro tests base class (#45761)

This commit moves many features of individual distro tests into the base
class so that other test cases can utilize them. It also standardizes
the pattern for tests adding assumptions for the particular
distributions to test.

qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -21,7 +21,6 @@ package org.elasticsearch.packaging.test;
 
 import org.apache.http.client.fluent.Request;
 import org.elasticsearch.packaging.util.Archives;
-import org.elasticsearch.packaging.util.Distribution;
 import org.elasticsearch.packaging.util.FileUtils;
 import org.elasticsearch.packaging.util.Installation;
 import org.elasticsearch.packaging.util.Platforms;
@@ -52,7 +51,6 @@ import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.Matchers.isEmptyString;
 import static org.junit.Assume.assumeThat;
 import static org.junit.Assume.assumeTrue;
@@ -60,9 +58,8 @@ import static org.junit.Assume.assumeTrue;
 public class ArchiveTests extends PackagingTestCase {
 
     @BeforeClass
-    public static void assumptions() {
-        assumeTrue("only archive distributions",
-            distribution().packaging == Distribution.Packaging.TAR || distribution().packaging == Distribution.Packaging.ZIP);
+    public static void filterDistros() {
+        assumeTrue("only archives", distribution.isArchive());
     }
 
     public void test10Install() throws Exception {
@@ -71,20 +68,14 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test20PluginsListWithNoPlugins() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         final Installation.Executables bin = installation.executables();
-        final Shell sh = newShell();
         final Result r = sh.run(bin.elasticsearchPlugin + " list");
 
         assertThat(r.stdout, isEmptyString());
     }
 
     public void test30NoJava() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         final Installation.Executables bin = installation.executables();
-        final Shell sh = newShell();
         sh.getEnv().remove("JAVA_HOME");
 
         final Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
@@ -105,10 +96,7 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test40CreateKeystoreManually() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         final Installation.Executables bin = installation.executables();
-        final Shell sh = newShell();
 
         Platforms.onLinux(() -> sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " create"));
 
@@ -138,12 +126,10 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test50StartAndStop() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         // cleanup from previous test
         rm(installation.config("elasticsearch.keystore"));
 
-        Archives.runElasticsearch(installation, newShell());
+        Archives.runElasticsearch(installation, sh);
 
         assertTrue("gc logs exist", Files.exists(installation.logs.resolve("gc.log")));
         ServerUtils.runElasticsearchTests();
@@ -152,8 +138,6 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void assertRunsWithJavaHome() throws Exception {
-        Shell sh = newShell();
-
         Platforms.onLinux(() -> {
             String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
             sh.getEnv().put("JAVA_HOME", systemJavaHome);
@@ -173,13 +157,10 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test51JavaHomeOverride() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         assertRunsWithJavaHome();
     }
 
     public void test52BundledJdkRemoved() throws Exception {
-        assumeThat(installation, is(notNullValue()));
         assumeThat(distribution().hasJdk, is(true));
 
         Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
@@ -192,8 +173,6 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test53JavaHomeWithSpecialCharacters() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         Platforms.onWindows(() -> {
             final Shell sh = new Shell();
             try {
@@ -247,13 +226,9 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test60AutoCreateKeystore() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660));
 
         final Installation.Executables bin = installation.executables();
-        final Shell sh = newShell();
-
         Platforms.onLinux(() -> {
             final Result result = sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " list");
             assertThat(result.stdout, containsString("keystore.seed"));
@@ -266,7 +241,6 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test70CustomPathConfAndJvmOptions() throws Exception {
-        assumeThat(installation, is(notNullValue()));
 
         final Path tempConf = getTempDir().resolve("esconf-alternate");
 
@@ -284,7 +258,6 @@ public class ArchiveTests extends PackagingTestCase {
                 "-Dlog4j2.disable.jmx=true\n";
             append(tempConf.resolve("jvm.options"), jvmOptions);
 
-            final Shell sh = newShell();
             Platforms.onLinux(() -> sh.run("chown -R elasticsearch:elasticsearch " + tempConf));
             Platforms.onWindows(() -> sh.run(
                 "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " +
@@ -297,11 +270,10 @@ public class ArchiveTests extends PackagingTestCase {
                 "}"
             ));
 
-            final Shell serverShell = newShell();
-            serverShell.getEnv().put("ES_PATH_CONF", tempConf.toString());
-            serverShell.getEnv().put("ES_JAVA_OPTS", "-XX:-UseCompressedOops");
+            sh.getEnv().put("ES_PATH_CONF", tempConf.toString());
+            sh.getEnv().put("ES_JAVA_OPTS", "-XX:-UseCompressedOops");
 
-            Archives.runElasticsearch(installation, serverShell);
+            Archives.runElasticsearch(installation, sh);
 
             final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
             assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
@@ -315,7 +287,6 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test80RelativePathConf() throws Exception {
-        assumeThat(installation, is(notNullValue()));
 
         final Path temp = getTempDir().resolve("esconf-alternate");
         final Path tempConf = temp.resolve("config");
@@ -330,7 +301,6 @@ public class ArchiveTests extends PackagingTestCase {
 
             append(tempConf.resolve("elasticsearch.yml"), "node.name: relative");
 
-            final Shell sh = newShell();
             Platforms.onLinux(() -> sh.run("chown -R elasticsearch:elasticsearch " + temp));
             Platforms.onWindows(() -> sh.run(
                 "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " +
@@ -343,10 +313,9 @@ public class ArchiveTests extends PackagingTestCase {
                 "}"
             ));
 
-            final Shell serverShell = newShell();
-            serverShell.setWorkingDirectory(temp);
-            serverShell.getEnv().put("ES_PATH_CONF", "config");
-            Archives.runElasticsearch(installation, serverShell);
+            sh.setWorkingDirectory(temp);
+            sh.getEnv().put("ES_PATH_CONF", "config");
+            Archives.runElasticsearch(installation, sh);
 
             final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
             assertThat(nodesResponse, containsString("\"name\":\"relative\""));
@@ -359,10 +328,7 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test90SecurityCliPackaging() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         final Installation.Executables bin = installation.executables();
-        final Shell sh = newShell();
 
         if (distribution().isDefault()) {
             assertTrue(Files.exists(installation.lib.resolve("tools").resolve("security-cli")));
@@ -383,10 +349,7 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test91ElasticsearchShardCliPackaging() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         final Installation.Executables bin = installation.executables();
-        final Shell sh = newShell();
 
         Platforms.PlatformAction action = () -> {
             final Result result = sh.run(bin.elasticsearchShard + " -h");
@@ -401,10 +364,7 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test92ElasticsearchNodeCliPackaging() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         final Installation.Executables bin = installation.executables();
-        final Shell sh = newShell();
 
         Platforms.PlatformAction action = () -> {
             final Result result = sh.run(bin.elasticsearchNode + " -h");
@@ -420,12 +380,9 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         Path relativeDataPath = installation.data.relativize(installation.home);
         append(installation.config("elasticsearch.yml"), "path.data: " + relativeDataPath);
 
-        final Shell sh = newShell();
         sh.setWorkingDirectory(getTempDir());
 
         Archives.runElasticsearch(installation, sh);
@@ -436,10 +393,7 @@ public class ArchiveTests extends PackagingTestCase {
     }
 
     public void test94ElasticsearchNodeExecuteCliNotEsHomeWorkDir() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         final Installation.Executables bin = installation.executables();
-        final Shell sh = newShell();
         // Run the cli tools from the tmp dir
         sh.setWorkingDirectory(getTempDir());
 

qa/os/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import junit.framework.TestCase;
+import org.elasticsearch.packaging.util.Distribution;
+import org.elasticsearch.packaging.util.FileUtils;
+import org.elasticsearch.packaging.util.Shell;
+import org.junit.Before;
+
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile;
+import static org.junit.Assume.assumeTrue;
+
+public class DebMetadataTests extends PackagingTestCase {
+
+    @Before
+    public void filterDistros() {
+        assumeTrue("only deb", distribution.packaging == Distribution.Packaging.DEB);
+    }
+
+    public void test05CheckLintian() {
+        sh.run("lintian --fail-on-warnings " + FileUtils.getDistributionFile(distribution()));
+    }
+
+    public void test06Dependencies() {
+
+        final Shell sh = new Shell();
+
+        final Shell.Result result = sh.run("dpkg -I " + getDistributionFile(distribution()));
+
+        TestCase.assertTrue(Pattern.compile("(?m)^ Depends:.*bash.*").matcher(result.stdout).find());
+
+        String oppositePackageName = "elasticsearch";
+        if (distribution().isDefault()) {
+            oppositePackageName += "-oss";
+        }
+
+        TestCase.assertTrue(Pattern.compile("(?m)^ Conflicts: " + oppositePackageName + "$").matcher(result.stdout).find());
+    }
+}

qa/os/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java

@@ -19,10 +19,8 @@
 
 package org.elasticsearch.packaging.test;
 
-import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
 import org.elasticsearch.packaging.util.Distribution;
-import org.elasticsearch.packaging.util.Shell;
-import org.junit.Before;
+import org.junit.BeforeClass;
 
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -32,37 +30,29 @@ import static org.elasticsearch.packaging.util.FileUtils.assertPathsExist;
 import static org.elasticsearch.packaging.util.Packages.SYSVINIT_SCRIPT;
 import static org.elasticsearch.packaging.util.Packages.assertInstalled;
 import static org.elasticsearch.packaging.util.Packages.assertRemoved;
-import static org.elasticsearch.packaging.util.Packages.install;
+import static org.elasticsearch.packaging.util.Packages.installPackage;
 import static org.elasticsearch.packaging.util.Packages.packageStatus;
 import static org.elasticsearch.packaging.util.Packages.remove;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
-import static org.elasticsearch.packaging.util.Platforms.isDPKG;
-import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.core.Is.is;
-import static org.junit.Assume.assumeThat;
 import static org.junit.Assume.assumeTrue;
 
-@TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
 public class DebPreservationTests extends PackagingTestCase {
 
-    @Before
-    public void onlyCompatibleDistributions() {
-        assumeTrue("only dpkg platforms", isDPKG());
-        assumeTrue("deb distributions", distribution().packaging == Distribution.Packaging.DEB);
-        assumeTrue("only bundled jdk", distribution().hasJdk);
-        assumeTrue("only compatible distributions", distribution().packaging.compatible);
+    @BeforeClass
+    public static void filterDistros() {
+        assumeTrue("only deb", distribution.packaging == Distribution.Packaging.DEB);
+        assumeTrue("only bundled jdk", distribution.hasJdk);
     }
 
     public void test10Install() throws Exception {
         assertRemoved(distribution());
-        installation = install(distribution());
+        installation = installPackage(distribution());
         assertInstalled(distribution());
         verifyPackageInstallation(installation, distribution(), newShell());
     }
 
     public void test20Remove() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         remove(distribution());
 
         // some config files were not removed
@@ -106,9 +96,6 @@ public class DebPreservationTests extends PackagingTestCase {
     }
 
     public void test30Purge() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
-        final Shell sh = new Shell();
         sh.run("dpkg --purge " + distribution().flavor.name);
 
         assertRemoved(distribution());

qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

@@ -19,15 +19,12 @@
 
 package org.elasticsearch.packaging.test;
 
-import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import org.apache.http.client.fluent.Request;
-import org.elasticsearch.packaging.util.Distribution;
 import org.elasticsearch.packaging.util.FileUtils;
-import org.elasticsearch.packaging.util.Shell;
 import org.elasticsearch.packaging.util.Shell.Result;
 import org.hamcrest.CoreMatchers;
-import org.junit.Before;
+import org.junit.BeforeClass;
 
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -50,53 +47,39 @@ import static org.elasticsearch.packaging.util.FileUtils.slurp;
 import static org.elasticsearch.packaging.util.Packages.SYSTEMD_SERVICE;
 import static org.elasticsearch.packaging.util.Packages.assertInstalled;
 import static org.elasticsearch.packaging.util.Packages.assertRemoved;
-import static org.elasticsearch.packaging.util.Packages.install;
+import static org.elasticsearch.packaging.util.Packages.installPackage;
 import static org.elasticsearch.packaging.util.Packages.remove;
 import static org.elasticsearch.packaging.util.Packages.restartElasticsearch;
 import static org.elasticsearch.packaging.util.Packages.startElasticsearch;
 import static org.elasticsearch.packaging.util.Packages.stopElasticsearch;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
 import static org.elasticsearch.packaging.util.Platforms.getOsRelease;
-import static org.elasticsearch.packaging.util.Platforms.isDPKG;
 import static org.elasticsearch.packaging.util.Platforms.isSystemd;
 import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
 import static org.elasticsearch.packaging.util.ServerUtils.runElasticsearchTests;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.isEmptyString;
 import static org.hamcrest.core.Is.is;
 import static org.junit.Assume.assumeThat;
 import static org.junit.Assume.assumeTrue;
 
-@TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
 public class PackageTests extends PackagingTestCase {
-    private Shell sh;
-
-    @Before
-    public void onlyCompatibleDistributions() throws Exception {
-        assumeTrue("only compatible distributions", distribution().packaging.compatible);
-        assumeTrue("rpm or deb",
-            distribution().packaging == Distribution.Packaging.DEB || distribution().packaging == Distribution.Packaging.RPM);
-        sh = newShell();
-    }
 
-    public void test05CheckLintian() throws Exception {
-        assumeTrue(isDPKG());
-        sh.run("lintian --fail-on-warnings " + FileUtils.getDistributionFile(distribution()));
+    @BeforeClass
+    public static void filterDistros() {
+        assumeTrue("rpm or deb", distribution.isPackage());
     }
 
     public void test10InstallPackage() throws Exception {
         assertRemoved(distribution());
-        installation = install(distribution());
+        installation = installPackage(distribution());
         assertInstalled(distribution());
         verifyPackageInstallation(installation, distribution(), sh);
     }
 
     public void test20PluginsCommandWhenNoPlugins() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         assertThat(sh.run(installation.bin("elasticsearch-plugin") + " list").stdout, isEmptyString());
     }
 
@@ -109,13 +92,10 @@ public class PackageTests extends PackagingTestCase {
     }
 
     public void test31InstallDoesNotStartServer() {
-        assumeThat(installation, is(notNullValue()));
-
         assertThat(sh.run("ps aux").stdout, not(containsString("org.elasticsearch.bootstrap.Elasticsearch")));
     }
 
     public void assertRunsWithJavaHome() throws Exception {
-        String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
         byte[] originalEnvFile = Files.readAllBytes(installation.envFile);
         try {
             Files.write(installation.envFile, ("JAVA_HOME=" + systemJavaHome + "\n").getBytes(StandardCharsets.UTF_8),
@@ -132,7 +112,6 @@ public class PackageTests extends PackagingTestCase {
     }
 
     public void test32JavaHomeOverride() throws Exception {
-        assumeThat(installation, is(notNullValue()));
         // we always run with java home when no bundled jdk is included, so this test would be repetitive
         assumeThat(distribution().hasJdk, is(true));
 
@@ -159,7 +138,6 @@ public class PackageTests extends PackagingTestCase {
     }
 
     public void test42BundledJdkRemoved() throws Exception {
-        assumeThat(installation, is(notNullValue()));
         assumeThat(distribution().hasJdk, is(true));
 
         Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
@@ -173,8 +151,6 @@ public class PackageTests extends PackagingTestCase {
 
     public void test40StartServer() throws Exception {
         String start = sh.runIgnoreExitCode("date ").stdout.trim();
-        assumeThat(installation, is(notNullValue()));
-
         startElasticsearch(sh);
 
         String journalEntries = sh.runIgnoreExitCode("journalctl _SYSTEMD_UNIT=elasticsearch.service " +
@@ -189,8 +165,6 @@ public class PackageTests extends PackagingTestCase {
     }
 
     public void test50Remove() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         // add fake bin directory as if a plugin was installed
         Files.createDirectories(installation.bin.resolve("myplugin"));
 
@@ -242,9 +216,7 @@ public class PackageTests extends PackagingTestCase {
     }
 
     public void test60Reinstall() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
-        installation = install(distribution());
+        installation = installPackage(distribution());
         assertInstalled(distribution());
         verifyPackageInstallation(installation, distribution(), sh);
 
@@ -254,7 +226,7 @@ public class PackageTests extends PackagingTestCase {
 
     public void test70RestartServer() throws Exception {
         try {
-            installation = install(distribution());
+            installation = installPackage(distribution());
             assertInstalled(distribution());
 
             startElasticsearch(sh);
@@ -269,7 +241,7 @@ public class PackageTests extends PackagingTestCase {
 
     public void test72TestRuntimeDirectory() throws Exception {
         try {
-            installation = install(distribution());
+            installation = installPackage(distribution());
             FileUtils.rm(installation.pidDir);
             startElasticsearch(sh);
             assertPathsExist(installation.pidDir);
@@ -280,7 +252,7 @@ public class PackageTests extends PackagingTestCase {
     }
 
     public void test73gcLogsExist() throws Exception {
-        installation = install(distribution());
+        installation = installPackage(distribution());
         startElasticsearch(sh);
         // it can be gc.log or gc.log.0.current
         assertThat(installation.logs, fileWithGlobExist("gc.log*"));
@@ -315,7 +287,6 @@ public class PackageTests extends PackagingTestCase {
     public void test81CustomPathConfAndJvmOptions() throws Exception {
         assumeTrue(isSystemd());
 
-        assumeThat(installation, is(notNullValue()));
         assertPathsExist(installation.envFile);
 
         stopElasticsearch(sh);
@@ -343,18 +314,17 @@ public class PackageTests extends PackagingTestCase {
 
             sh.runIgnoreExitCode("chown -R elasticsearch:elasticsearch " + tempConf);
 
-            final Shell serverShell = newShell();
             cp(installation.envFile, tempConf.resolve("elasticsearch.bk"));//backup
             append(installation.envFile, "ES_PATH_CONF=" + tempConf + "\n");
             append(installation.envFile, "ES_JAVA_OPTS=-XX:-UseCompressedOops");
 
-            startElasticsearch(serverShell);
+            startElasticsearch(sh);
 
             final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
             assertThat(nodesResponse, CoreMatchers.containsString("\"heap_init_in_bytes\":536870912"));
             assertThat(nodesResponse, CoreMatchers.containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
 
-            stopElasticsearch(serverShell);
+            stopElasticsearch(sh);
 
         } finally {
             rm(installation.envFile);
@@ -370,7 +340,7 @@ public class PackageTests extends PackagingTestCase {
 
             sh.run("systemctl mask systemd-sysctl.service");
 
-            installation = install(distribution());
+            installation = installPackage(distribution());
 
             sh.run("systemctl unmask systemd-sysctl.service");
         } finally {
@@ -382,7 +352,7 @@ public class PackageTests extends PackagingTestCase {
         // Limits are changed on systemd platforms only
         assumeTrue(isSystemd());
 
-        installation = install(distribution());
+        installation = installPackage(distribution());
 
         startElasticsearch(sh);
 

qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -32,50 +32,86 @@ import org.elasticsearch.packaging.util.Shell;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.rules.TestName;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
 import org.junit.runner.RunWith;
 
 import java.nio.file.Paths;
 
 import static org.elasticsearch.packaging.util.Cleanup.cleanEverything;
+import static org.junit.Assume.assumeFalse;
 import static org.junit.Assume.assumeTrue;
 
+/**
+ * Class that all packaging test cases should inherit from
+ */
 @RunWith(RandomizedRunner.class)
 @TestMethodProviders({
     JUnit3MethodProvider.class
 })
 @TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
-/**
- * Class that all packaging test cases should inherit from. This makes working with the packaging tests more similar to what we're
- * familiar with from {@link org.elasticsearch.test.ESTestCase} without having to apply its behavior that's not relevant here
- */
 public abstract class PackagingTestCase extends Assert {
 
     protected final Log logger = LogFactory.getLog(getClass());
 
-    private static Distribution distribution;
+    // the distribution being tested
+    protected static final Distribution distribution;
     static {
         distribution = new Distribution(Paths.get(System.getProperty("tests.distribution")));
     }
 
+    // the java installation already installed on the system
+    protected static final String systemJavaHome;
+    static {
+        Shell sh = new Shell();
+        if (Platforms.LINUX) {
+            systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
+        } else {
+            assert Platforms.WINDOWS;
+            systemJavaHome = sh.run("$Env:SYSTEM_JAVA_HOME").stdout.trim();
+        }
+    }
+
+    // the current installation of the distribution being tested
+    protected static Installation installation;
+
+    private static boolean failed;
+
+    @ClassRule
+    public static final TestWatcher testFailureRule = new TestWatcher() {
+        @Override
+        protected void failed(Throwable e, Description description) {
+            failed = true;
+        }
+    };
+
+    // a shell to run system commands with
+    protected Shell sh;
+
     @Rule
     public final TestName testNameRule = new TestName();
 
-    @Before
-    public void setup() {
-        assumeTrue("only compatible distributions", distribution().packaging.compatible);
-        logger.info("[" + testNameRule.getMethodName() + "]: before test");
+    @BeforeClass
+    public static void filterCompatible() {
+        assumeTrue("only compatible distributions", distribution.packaging.compatible);
     }
 
-    protected static Installation installation;
-
     @BeforeClass
     public static void cleanup() throws Exception {
         installation = null;
         cleanEverything();
     }
 
+    @Before
+    public void setup() throws Exception {
+        assumeFalse(failed); // skip rest of tests once one fails
+
+        sh = newShell();
+    }
+
     /** The {@link Distribution} that should be tested in this case */
     protected static Distribution distribution() {
         return distribution;
@@ -85,11 +121,9 @@ public abstract class PackagingTestCase extends Assert {
         Shell sh = new Shell();
         if (distribution().hasJdk == false) {
             Platforms.onLinux(() -> {
-                String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
                 sh.getEnv().put("JAVA_HOME", systemJavaHome);
             });
             Platforms.onWindows(() -> {
-                final String systemJavaHome = sh.run("$Env:SYSTEM_JAVA_HOME").stdout.trim();
                 sh.getEnv().put("JAVA_HOME", systemJavaHome);
             });
         }

qa/os/src/test/java/org/elasticsearch/packaging/test/PackageConflictTests.java → qa/os/src/test/java/org/elasticsearch/packaging/test/RpmMetadataTests.java

@@ -30,37 +30,14 @@ import java.util.regex.Pattern;
 import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile;
 import static org.junit.Assume.assumeTrue;
 
-public class PackageConflictTests extends PackagingTestCase {
-
-    private Shell sh;
+public class RpmMetadataTests extends PackagingTestCase {
 
     @Before
-    public void onlyCompatibleDistributions() throws Exception {
-        assumeTrue("only compatible distributions", distribution().packaging.compatible);
-        assumeTrue("rpm or deb",
-            distribution().packaging == Distribution.Packaging.DEB || distribution().packaging == Distribution.Packaging.RPM);
-        sh = newShell();
-    }
-
-    public void test11DebDependencies() {
-        // TODO: rewrite this test to not use a real second distro to try and install
-        assumeTrue(Platforms.isDPKG());
-
-        final Shell sh = new Shell();
-
-        final Shell.Result result = sh.run("dpkg -I " + getDistributionFile(distribution()));
-
-        TestCase.assertTrue(Pattern.compile("(?m)^ Depends:.*bash.*").matcher(result.stdout).find());
-
-        String oppositePackageName = "elasticsearch";
-        if (distribution().isDefault()) {
-            oppositePackageName += "-oss";
-        }
-
-        TestCase.assertTrue(Pattern.compile("(?m)^ Conflicts: " + oppositePackageName + "$").matcher(result.stdout).find());
+    public void filterDistros() {
+        assumeTrue("only rpm", distribution.packaging == Distribution.Packaging.RPM);
     }
 
-    public void test11RpmDependencies() {
+    public void test11Dependencies() {
         // TODO: rewrite this test to not use a real second distro to try and install
         assumeTrue(Platforms.isRPM());
 

qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java

@@ -19,10 +19,9 @@
 
 package org.elasticsearch.packaging.test;
 
-import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
 import org.elasticsearch.packaging.util.Distribution;
 import org.elasticsearch.packaging.util.Shell;
-import org.junit.Before;
+import org.junit.BeforeClass;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -34,37 +33,29 @@ import static org.elasticsearch.packaging.util.Packages.SYSTEMD_SERVICE;
 import static org.elasticsearch.packaging.util.Packages.SYSVINIT_SCRIPT;
 import static org.elasticsearch.packaging.util.Packages.assertInstalled;
 import static org.elasticsearch.packaging.util.Packages.assertRemoved;
-import static org.elasticsearch.packaging.util.Packages.install;
+import static org.elasticsearch.packaging.util.Packages.installPackage;
 import static org.elasticsearch.packaging.util.Packages.remove;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
-import static org.elasticsearch.packaging.util.Platforms.isRPM;
 import static org.elasticsearch.packaging.util.Platforms.isSystemd;
-import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.core.Is.is;
-import static org.junit.Assume.assumeThat;
 import static org.junit.Assume.assumeTrue;
 
-@TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
 public class RpmPreservationTests extends PackagingTestCase {
 
-    @Before
-    public void onlyCompatibleDistributions() {
-        assumeTrue("only rpm platforms", isRPM());
-        assumeTrue("rpm distributions", distribution().packaging == Distribution.Packaging.RPM);
+    @BeforeClass
+    public static void filterDistros() {
+        assumeTrue("only rpm", distribution.packaging == Distribution.Packaging.RPM);
         assumeTrue("only bundled jdk", distribution().hasJdk);
-        assumeTrue("only compatible distributions", distribution().packaging.compatible);
     }
 
     public void test10Install() throws Exception {
         assertRemoved(distribution());
-        installation = install(distribution());
+        installation = installPackage(distribution());
         assertInstalled(distribution());
         verifyPackageInstallation(installation, distribution(), newShell());
     }
 
     public void test20Remove() throws Exception {
-        assumeThat(installation, is(notNullValue()));
-
         remove(distribution());
 
         // config was removed
@@ -80,7 +71,7 @@ public class RpmPreservationTests extends PackagingTestCase {
     public void test30PreserveConfig() throws Exception {
         final Shell sh = new Shell();
 
-        installation = install(distribution());
+        installation = installPackage(distribution());
         assertInstalled(distribution());
         verifyPackageInstallation(installation, distribution(), newShell());
 

qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java

@@ -26,7 +26,6 @@ import org.elasticsearch.packaging.util.ServerUtils;
 import org.elasticsearch.packaging.util.Shell;
 import org.elasticsearch.packaging.util.Shell.Result;
 import org.junit.After;
-import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
@@ -47,13 +46,6 @@ public class WindowsServiceTests extends PackagingTestCase {
     private static final String DEFAULT_DISPLAY_NAME = "Elasticsearch " + FileUtils.getCurrentVersion() + " (elasticsearch-service-x64)";
     private static String serviceScript;
 
-    private Shell sh;
-
-    @Before
-    public void createShell() {
-        sh = new Shell();
-    }
-
     @BeforeClass
     public static void ensureWindows() {
         assumeTrue(Platforms.WINDOWS);

qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java

@@ -49,6 +49,14 @@ public class Distribution {
         return flavor.equals(Flavor.OSS);
     }
 
+    public boolean isArchive() {
+        return packaging == Packaging.TAR || packaging == Packaging.ZIP;
+    }
+
+    public boolean isPackage() {
+        return packaging == Packaging.RPM || packaging == Packaging.DEB;
+    }
+
     public enum Packaging {
 
         TAR(".tar.gz", Platforms.LINUX || Platforms.DARWIN),

qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java

@@ -94,7 +94,7 @@ public class Packages {
         return result;
     }
 
-    public static Installation install(Distribution distribution) throws IOException {
+    public static Installation installPackage(Distribution distribution) throws IOException {
         Shell sh = new Shell();
         String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
         if (distribution.hasJdk == false) {