Convert die with dignity test to new rest test infra (#97734)

This commit rewrites the DieWithDignity test to use the new test infra.
A side effect of this change is that it no longer relies on jps, which
appears to have issues on Windows.

closes #77282

test/external-modules/die-with-dignity/build.gradle

@@ -1,7 +1,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.util.GradleUtils
 
-apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
@@ -12,21 +12,6 @@ esplugin {
 // let the javaRestTest see the classpath of main
 GradleUtils.extendSourceSet(project, "main", "javaRestTest", tasks.named("javaRestTest"))
 
-tasks.named("javaRestTest").configure {
-  it.onlyIf("snapshot build") { BuildParams.isSnapshotBuild() }
-  systemProperty 'tests.security.manager', 'false'
-  systemProperty 'tests.system_call_filter', 'false'
-  nonInputProperties.systemProperty 'log', testClusters.named("javaRestTest").map(c -> c.singleNode().getServerLog())
-  systemProperty 'runtime.java.home', BuildParams.runtimeJavaHome
-}
-
-testClusters.matching { it.name == "javaRestTest" }.configureEach {
-  systemProperty "die.with.dignity.test", "true"
-  // disable exit on out of memory error to let DieWithDignityIT verify that OOM handling without that works (including OOMs that are not caused by
-  // memory like native threads. We leave it to the JVM to test that exit on OOM works via the flag.
-  jvmArgs '-XX:-ExitOnOutOfMemoryError'
-}
-
 tasks.named("test").configure {
   enabled = false
 }

test/external-modules/die-with-dignity/src/javaRestTest/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java

@@ -8,10 +8,14 @@
 
 package org.elasticsearch.qa.die_with_dignity;
 
-import org.apache.lucene.util.Constants;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.LogType;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.ESRestTestCase;
+import org.hamcrest.Matcher;
+import org.junit.ClassRule;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -20,110 +24,84 @@ import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
 
 public class DieWithDignityIT extends ESRestTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.INTEG_TEST)
+        .module("test-die-with-dignity")
+        .setting("xpack.security.enabled", "false")
+        .environment("CLI_JAVA_OPTS", "-Ddie.with.dignity.test=true")
+        .jvmArg("-XX:-ExitOnOutOfMemoryError")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     public void testDieWithDignity() throws Exception {
-        assumeFalse("Mute on Windows, see https://github.com/elastic/elasticsearch/issues/77282", Constants.WINDOWS);
-        // there should be an Elasticsearch process running with the die.with.dignity.test system property
-        {
-            final Map<String, String> esCommandLines = getElasticsearchCommandLines();
-            final boolean found = esCommandLines.values().stream().anyMatch(line -> line.contains("-Ddie.with.dignity.test=true"));
-            assertTrue(esCommandLines.toString(), found);
-        }
+        final long pid = cluster.getPid(0);
+        assertJvmArgs(pid, containsString("-Ddie.with.dignity.test=true"));
 
         expectThrows(IOException.class, () -> client().performRequest(new Request("GET", "/_die_with_dignity")));
 
-        // the Elasticsearch process should die and disappear from the output of jps
-        assertBusy(() -> {
-            final Map<String, String> esCommandLines = getElasticsearchCommandLines();
-            final boolean notFound = esCommandLines.values().stream().noneMatch(line -> line.contains("-Ddie.with.dignity.test=true"));
-            assertTrue(esCommandLines.toString(), notFound);
-        });
+        // the Elasticsearch process should die
+        assertBusy(() -> assertJvmArgs(pid, not(containsString("-Ddie.with.dignity.test=true"))));
 
         // parse the logs and ensure that Elasticsearch died with the expected cause
-        final List<String> lines = Files.readAllLines(PathUtils.get(System.getProperty("log")));
-        final Iterator<String> it = lines.iterator();
-
         boolean fatalError = false;
         boolean fatalErrorInThreadExiting = false;
+        for (String line : readLines(cluster.getNodeLog(0, LogType.SERVER_JSON))) {
+            if (containsAll(line, ".*ERROR.*", ".*ExceptionsHelper.*", ".*fatal error.*")) {
+                fatalError = true;
+            } else if (containsAll(
+                line,
+                ".*ERROR.*",
+                ".*ElasticsearchUncaughtExceptionHandler.*",
+                ".*fatal error in thread \\[Thread-\\d+\\], exiting.*",
+                ".*java.lang.OutOfMemoryError: Requested array size exceeds VM limit.*"
+            )) {
+                fatalErrorInThreadExiting = true;
+            }
+        }
+        assertTrue(fatalError);
+        assertTrue(fatalErrorInThreadExiting);
+    }
+
+    private void assertJvmArgs(long pid, Matcher<String> matcher) throws IOException {
+        final String jcmdPath = PathUtils.get(System.getProperty("tests.runtime.java"), "bin/jcmd").toString();
+        final Process jcmdProcess = new ProcessBuilder().command(jcmdPath, Long.toString(pid), "VM.command_line")
+            .redirectErrorStream(true)
+            .start();
+        List<String> outputLines = readLines(jcmdProcess.getInputStream());
+
+        String jvmArgs = null;
         try {
-            while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) {
-                final String line = it.next();
-                if (containsAll(line, ".*ERROR.*", ".*ExceptionsHelper.*", ".*javaRestTest-0.*", ".*fatal error.*")) {
-                    fatalError = true;
-                } else if (containsAll(
-                    line,
-                    ".*ERROR.*",
-                    ".*ElasticsearchUncaughtExceptionHandler.*",
-                    ".*javaRestTest-0.*",
-                    ".*fatal error in thread \\[Thread-\\d+\\], exiting.*",
-                    ".*java.lang.OutOfMemoryError: Requested array size exceeds VM limit.*"
-                )) {
-                    fatalErrorInThreadExiting = true;
+            for (String line : outputLines) {
+                if (line.startsWith("jvm_args")) {
+                    jvmArgs = line;
+                    break;
                 }
             }
-
-            assertTrue(fatalError);
-            assertTrue(fatalErrorInThreadExiting);
+            assertThat(jvmArgs, matcher);
 
         } catch (AssertionError ae) {
-            Path path = PathUtils.get(System.getProperty("log"));
-            debugLogs(path);
+            logger.error("Failed matcher for jvm pid " + pid);
+            logger.error("jcmd output: " + String.join("\n", outputLines));
             throw ae;
         }
     }
 
-    private Map<String, String> getElasticsearchCommandLines() throws IOException {
-        /*
-         * jps will truncate the command line to 1024 characters; so we collect the pids and then run jcmd <pid> VM.command_line to get the
-         * full command line.
-         */
-        final String jpsPath = PathUtils.get(System.getProperty("runtime.java.home"), "bin/jps").toString();
-        final Process process = new ProcessBuilder().command(jpsPath, "-q").start();
-
-        final List<String> pids = new ArrayList<>();
-        try (
-            InputStream is = process.getInputStream();
-            BufferedReader in = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))
-        ) {
-            String line;
-            while ((line = in.readLine()) != null) {
-                pids.add(line);
-            }
-        }
-
-        final String jcmdPath = PathUtils.get(System.getProperty("runtime.java.home"), "bin/jcmd").toString();
-        final Map<String, String> esCommandLines = new HashMap<>();
-        for (final String pid : pids) {
-            final Process jcmdProcess = new ProcessBuilder().command(jcmdPath, pid, "VM.command_line").start();
-            try (
-                InputStream is = jcmdProcess.getInputStream();
-                BufferedReader in = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))
-            ) {
-                boolean isElasticsearch = false;
-                String jvmArgs = null;
-                String line;
-                while ((line = in.readLine()) != null) {
-                    if (line.equals("java_command: org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch")) {
-                        isElasticsearch = true;
-                    }
-                    if (line.startsWith("jvm_args")) {
-                        jvmArgs = line;
-                    }
-                }
-                if (isElasticsearch) {
-                    assertNotNull(pid, jvmArgs);
-                    esCommandLines.put(pid, jvmArgs);
-                }
-            }
+    private List<String> readLines(InputStream is) throws IOException {
+        try (BufferedReader in = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
+            return in.lines().toList();
         }
-        return esCommandLines;
     }
 
     private boolean containsAll(String line, String... subStrings) {