Make EnrollmentProcessTests more robust v2 (#83507)

In the Docker test, try to bullet-proof the call to
`elasticsearch-create-enrollment-token` by:

   1. Checking the container logs for the node starting up
   2. Don't all the CLI call to automatically fail, instead manually
      check the result and throw an assertion failure if the call failed

qa/os/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java

@@ -94,20 +94,12 @@ public class EnrollmentProcessTests extends PackagingTestCase {
         waitForElasticsearch(installation);
         final String node1ContainerId = Docker.getContainerId();
 
-        final AtomicReference<String> enrollmentToken = new AtomicReference<>();
+        Docker.waitForNodeStarted(node1ContainerId);
 
-        assertBusy(() -> {
-            final String tokenValue = installation.executables().createEnrollmentToken.run("-s node")
-                .stdout()
-                .lines()
-                .filter(line -> line.startsWith("WARNING:") == false)
-                .findFirst()
-                .orElseThrow(() -> new AssertionError("Failed to find any non-warning output lines"));
-            enrollmentToken.set(tokenValue);
-        }, 30, TimeUnit.SECONDS);
+        String enrollmentToken = getEnrollmentToken();
 
         // installation refers to second node from now on
-        installation = runAdditionalContainer(distribution(), builder().envVar("ENROLLMENT_TOKEN", enrollmentToken.get()), 9201, 9301);
+        installation = runAdditionalContainer(distribution(), builder().envVar("ENROLLMENT_TOKEN", enrollmentToken), 9201, 9301);
 
         // TODO Make our packaging test methods aware of multiple installations, see https://github.com/elastic/elasticsearch/issues/79688
         waitForElasticsearch(installation);
@@ -130,6 +122,34 @@ public class EnrollmentProcessTests extends PackagingTestCase {
         removeContainer(node1ContainerId);
     }
 
+    private String getEnrollmentToken() throws Exception {
+        final AtomicReference<String> enrollmentTokenHolder = new AtomicReference<>();
+
+        assertBusy(() -> {
+            // `assertBusy` only retries on assertion errors, not exceptions, and `Executable#run(String)`
+            // throws a `Shell.ShellException` if the command isn't successful.
+            final Shell.Result result = installation.executables().createEnrollmentToken.run("-s node", null, true);
+
+            if (result.isSuccess() == false) {
+                if (result.stdout().contains("Failed to determine the health of the cluster")) {
+                    throw new AssertionError("Elasticsearch is not ready yet");
+                }
+                throw new Shell.ShellException(
+                    "Command was not successful: [elasticsearch-create-enrollment-token -s node]\n   result: " + result
+                );
+            }
+
+            final String tokenValue = result.stdout()
+                .lines()
+                .filter(line -> line.startsWith("WARNING:") == false)
+                .findFirst()
+                .orElseThrow(() -> new AssertionError("Failed to find any non-warning output lines"));
+            enrollmentTokenHolder.set(tokenValue);
+        }, 30, TimeUnit.SECONDS);
+
+        return enrollmentTokenHolder.get();
+    }
+
     private void waitForSecondNode() {
         int retries = 60;
         while (retries > 0) {

qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java

@@ -140,7 +140,7 @@ public class Shell {
         logger.warn("Running command with env: " + env);
         Result result = runScriptIgnoreExitCode(command);
         if (result.isSuccess() == false) {
-            throw new ShellException("Command was not successful: [" + String.join(" ", command) + "]\n   result: " + result.toString());
+            throw new ShellException("Command was not successful: [" + String.join(" ", command) + "]\n   result: " + result);
         }
         return result;
     }

qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.packaging.util.docker;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
@@ -37,10 +38,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static java.nio.file.attribute.PosixFilePermissions.fromString;
+import static org.elasticsearch.packaging.test.PackagingTestCase.assertBusy;
 import static org.elasticsearch.packaging.util.FileMatcher.Fileness.Directory;
 import static org.elasticsearch.packaging.util.FileMatcher.p444;
 import static org.elasticsearch.packaging.util.FileMatcher.p555;
@@ -677,6 +680,10 @@ public class Docker {
     }
 
     public static Shell.Result getContainerLogs() {
+        return getContainerLogs(containerId);
+    }
+
+    public static Shell.Result getContainerLogs(String containerId) {
         return sh.run("docker logs " + containerId);
     }
 
@@ -721,7 +728,34 @@ public class Docker {
         return dockerShell.run("ls -1 --color=never " + path).stdout().lines().collect(Collectors.toList());
     }
 
+    /**
+     * Returns a list of the file contents of the supplied path.
+     * @param path the path to list
+     * @return the listing
+     */
     public static List<String> listContents(Path path) {
         return listContents(path.toString());
     }
+
+    /**
+     * Waits for an Elasticsearch node start by looking at the cluster logs. This is useful if the
+     * container is not available on an external port, or authentication is in force and credentials
+     * are not available.
+     * @param containerId the container to check
+     */
+    public static void waitForNodeStarted(String containerId) throws Exception {
+        ObjectMapper mapper = new ObjectMapper();
+        // Some lines are not JSON, filter those out
+        assertBusy(() -> assertTrue(getContainerLogs(containerId).stdout().lines().filter(line -> line.startsWith("{")).map(line -> {
+            try {
+                return mapper.readTree(line);
+            } catch (JsonProcessingException e) {
+                throw new RuntimeException(e);
+            }
+        })
+            .anyMatch(
+                json -> json.get("message").textValue().contains("started")
+                    && json.get("log.logger").textValue().equals("org.elasticsearch.node.Node")
+            )), 60, TimeUnit.SECONDS);
+    }
 }