Fix DieWithDignity logs assertion (#43543)

This test is likely to kill the server in the middle of writing logs. This means that we can end up with logs with partially written json log lines and standard json parsers would fail on this.
This fix is to use regular expressions on json logs.(just like the previous approach on plain text logs)
closes #43413

qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java

@@ -21,14 +21,10 @@ package org.elasticsearch.qa.die_with_dignity;
 
 import org.apache.http.ConnectionClosedException;
 import org.apache.lucene.util.Constants;
-import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.logging.JsonLogLine;
-import org.elasticsearch.common.logging.JsonLogsStream;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.hamcrest.Matcher;
-import org.hamcrest.Matchers;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -38,12 +34,10 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Iterator;
 import java.util.List;
-import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
@@ -91,29 +85,29 @@ public class DieWithDignityIT extends ESRestTestCase {
             }
         });
 
-        try {
-            // parse the logs and ensure that Elasticsearch died with the expected cause
-            Path path = PathUtils.get(System.getProperty("log"));
-            try (Stream<JsonLogLine> stream = JsonLogsStream.from(path)) {
-                final Iterator<JsonLogLine> it = stream.iterator();
+        // parse the logs and ensure that Elasticsearch died with the expected cause
+        final List<String> lines = Files.readAllLines(PathUtils.get(System.getProperty("log")));
 
-                boolean fatalError = false;
-                boolean fatalErrorInThreadExiting = false;
+        final Iterator<String> it = lines.iterator();
 
-                while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) {
-                    final JsonLogLine line = it.next();
-                    if (isFatalError(line)) {
-                        fatalError = true;
-                    } else if (isFatalErrorInThreadExiting(line) || isWarnExceptionReceived(line)) {
-                        fatalErrorInThreadExiting = true;
-                        assertThat(line.stacktrace(),
-                            hasItem(Matchers.containsString("java.lang.OutOfMemoryError: die with dignity")));
-                    }
+        boolean fatalError = false;
+        boolean fatalErrorInThreadExiting = false;
+        try {
+            while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) {
+                final String line = it.next();
+                if (line.matches(".*ERROR.*o\\.e\\.ExceptionsHelper.*node-0.*fatal error.*")) {
+                    fatalError = true;
+                } else if (line.matches(".*ERROR.*o\\.e\\.b\\.ElasticsearchUncaughtExceptionHandler.*node-0.*"
+                    + "fatal error in thread \\[Thread-\\d+\\], exiting.*")) {
+                    fatalErrorInThreadExiting = true;
+                    assertTrue(it.hasNext());
+                    assertThat(it.next(), containsString("java.lang.OutOfMemoryError: die with dignity"));
                 }
-
-                assertTrue(fatalError);
-                assertTrue(fatalErrorInThreadExiting);
             }
+
+            assertTrue(fatalError);
+            assertTrue(fatalErrorInThreadExiting);
+
         } catch (AssertionError ae) {
             Path path = PathUtils.get(System.getProperty("log"));
             debugLogs(path);
@@ -121,34 +115,12 @@ public class DieWithDignityIT extends ESRestTestCase {
         }
     }
 
-    private boolean isWarnExceptionReceived(JsonLogLine line) {
-        return line.level().equals("WARN")
-            && line.component().equals("o.e.h.AbstractHttpServerTransport")
-            && line.nodeName().equals("node-0")
-            && line.message().contains("caught exception while handling client http traffic");
-    }
-
     private void debugLogs(Path path) throws IOException {
         try (BufferedReader reader = Files.newBufferedReader(path)) {
-            Terminal terminal = Terminal.DEFAULT;
-            reader.lines().forEach(line -> terminal.println(line));
+            reader.lines().forEach(line -> logger.info(line));
         }
     }
 
-    private boolean isFatalErrorInThreadExiting(JsonLogLine line) {
-        return line.level().equals("ERROR")
-            && line.component().equals("o.e.b.ElasticsearchUncaughtExceptionHandler")
-            && line.nodeName().equals("node-0")
-            && line.message().matches("fatal error in thread \\[Thread-\\d+\\], exiting$");
-    }
-
-    private boolean isFatalError(JsonLogLine line) {
-        return line.level().equals("ERROR")
-            && line.component().equals("o.e.ExceptionsHelper")
-            && line.nodeName().equals("node-0")
-            && line.message().contains("fatal error");
-    }
-
     @Override
     protected boolean preserveClusterUponCompletion() {
         // as the cluster is dead its state can not be wiped successfully so we have to bypass wiping the cluster