Rerun test task when test jdk crashed with System exit (#71881)

Related to #52610 this PR introduces a rerun of all tests for a test task if the test jvm 
has crashed because of a system exit. We furthermore log potential tests that caused 
the System.exit based on which tests have been active at the time of the system exit.

We also modified the build scan logic to track unexpected test jvm exists 
with the tag `unexpected-test-jvm-exit`

build.gradle

@@ -56,6 +56,8 @@ if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
 String elasticLicenseUrl = "https://raw.githubusercontent.com/elastic/elasticsearch/${licenseCommit}/licenses/ELASTIC-LICENSE-2.0.txt"
 
 subprojects {
+  apply plugin:'elasticsearch.internal-test-rerun'
+
   // Default to the SSPL+Elastic dual license
   project.ext.projectLicenses = [
     'Server Side Public License, v 1': 'https://www.mongodb.com/licensing/server-side-public-license',
@@ -102,6 +104,7 @@ subprojects {
     project.licenseFile = project.rootProject.file('licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
     project.noticeFile = project.rootProject.file('NOTICE.txt')
   }
+
 }
 
 /**

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

@@ -67,6 +67,7 @@ abstract class AbstractGradleFuncTest extends Specification {
         return input.readLines()
                 .collect { it.replace('\\', '/') }
                 .collect {it.replace(normalizedPathPrefix , '.') }
+                .collect {it.replaceAll(/Gradle Test Executor \d/ , 'Gradle Test Executor 1') }
                 .join("\n")
     }
 

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rerun/InternalTestRerunPluginFuncTest.groovy

@@ -0,0 +1,242 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rerun
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+
+class InternalTestRerunPluginFuncTest extends AbstractGradleFuncTest {
+
+    def "does not rerun on failed tests"() {
+        when:
+        buildFile.text = """
+        plugins {
+          id 'java'
+          id 'elasticsearch.internal-test-rerun'
+        }
+
+        repositories {
+            mavenCentral()
+        }
+        
+        dependencies {
+            testImplementation 'junit:junit:4.13.1'
+        }
+        
+        tasks.named("test").configure {
+            maxParallelForks = 4
+            testLogging {
+                events "standard_out", "failed"
+                exceptionFormat "short"
+            }
+        }
+        
+        """
+        createTest("SimpleTest")
+        createTest("SimpleTest2")
+        createTest("SimpleTest3")
+        createTest("SimpleTest4")
+        createTest("SimpleTest5")
+        createFailedTest("SimpleTest6")
+        createFailedTest("SimpleTest7")
+        createFailedTest("SimpleTest8")
+        createTest("SomeOtherTest")
+        createTest("SomeOtherTest1")
+        createTest("SomeOtherTest2")
+        createTest("SomeOtherTest3")
+        createTest("SomeOtherTest4")
+        createTest("SomeOtherTest5")
+        then:
+        def result = gradleRunner("test").buildAndFail()
+        result.output.contains("total executions: 2") == false
+        and: "no jvm system exit tracing provided"
+        normalized(result.output).contains("""Test jvm exited unexpectedly.
+Test jvm system exit trace:""") == false
+    }
+
+    def "all tests are rerun when test jvm has crashed"() {
+        when:
+        settingsFile.text = """
+        plugins {
+            id "com.gradle.enterprise" version "3.6.1"
+        }
+        gradleEnterprise {
+            server = 'https://gradle-enterprise.elastic.co/'
+        }
+        """ + settingsFile.text
+
+        buildFile.text = """
+        plugins {
+          id 'java'
+          id 'elasticsearch.internal-test-rerun'
+        }
+
+        repositories {
+            mavenCentral()
+        }
+        
+        dependencies {
+            testImplementation 'junit:junit:4.13.1'
+        }
+        
+        tasks.named("test").configure {
+            maxParallelForks = 4
+            testLogging {
+                // set options for log level LIFECYCLE
+                events "started", "passed", "standard_out", "failed"
+                exceptionFormat "short"
+            }
+        }
+        
+        """
+        createTest("AnotherTest")
+        createFailedTest("AnotherTest1")
+        createTest("AnotherTest2")
+        createTest("AnotherTest3")
+        createTest("AnotherTest4")
+        createTest("AnotherTest5")
+        createSystemExitTest("AnotherTest6")
+        createTest("AnotherTest7")
+        createTest("AnotherTest8")
+        createFailedTest("AnotherTest9")
+        createTest("AnotherTest10")
+        createTest("SimpleTest")
+        createTest("SimpleTest2")
+        createTest("SimpleTest3")
+        createTest("SimpleTest4")
+        createTest("SimpleTest5")
+        createTest("SimpleTest6")
+        createTest("SimpleTest7")
+        createTest("SimpleTest8")
+        createTest("SomeOtherTest")
+        then:
+        def result = gradleRunner("test").build()
+        result.output.contains("AnotherTest6 total executions: 2")
+        // triggered only in the second overall run
+        and: 'Tracing is provided'
+        normalized(result.output).contains("""================
+Test jvm exited unexpectedly.
+Test jvm system exit trace (run: 1)
+Gradle Test Executor 1 > AnotherTest6 > someTest
+================""")
+    }
+
+    def "reruns tests till max rerun count is reached"() {
+        when:
+        buildFile.text = """
+        plugins {
+          id 'java'
+          id 'elasticsearch.internal-test-rerun'
+        }
+
+        repositories {
+            mavenCentral()
+        }
+        
+        dependencies {
+            testImplementation 'junit:junit:4.13.1'
+        }
+        
+        tasks.named("test").configure {
+            rerun {
+                maxReruns = 4
+            }
+            testLogging {
+                // set options for log level LIFECYCLE
+                events "standard_out", "failed"
+                exceptionFormat "short"
+            }
+        }
+        """
+        createSystemExitTest("JdkKillingTest", 5)
+        then:
+        def result = gradleRunner("test").buildAndFail()
+        result.output.contains("JdkKillingTest total executions: 4")
+        result.output.contains("Max retries(4) hit")
+        and: 'Tracing is provided'
+        normalized(result.output).contains("Test jvm system exit trace (run: 1)")
+        normalized(result.output).contains("Test jvm system exit trace (run: 2)")
+        normalized(result.output).contains("Test jvm system exit trace (run: 3)")
+        normalized(result.output).contains("Test jvm system exit trace (run: 4)")
+    }
+
+    private String testMethodContent(boolean withSystemExit, boolean fail, int timesFailing = 1) {
+        return """
+            int count = countExecutions();
+            System.out.println(getClass().getSimpleName() + " total executions: " + count);
+
+            ${withSystemExit ? """
+                    if(count <= ${timesFailing}) {
+                        System.exit(1);
+                    }
+                    """ : ''
+            }
+
+            ${fail ? """
+                    if(count <= ${timesFailing}) {
+                        Assert.fail();
+                    }
+                    """ : ''
+            }
+        """
+    }
+
+    private File createSystemExitTest(String clazzName, timesFailing = 1) {
+        createTest(clazzName, testMethodContent(true, false, timesFailing))
+    }
+    private File createFailedTest(String clazzName) {
+        createTest(clazzName, testMethodContent(false, true, 1))
+    }
+
+    private File createTest(String clazzName, String content = testMethodContent(false, false, 1)) {
+        file("src/test/java/org/acme/${clazzName}.java") << """
+            import org.junit.Test;
+            import org.junit.Before;
+            import org.junit.After;
+            import org.junit.Assert;
+            import java.nio.*;
+            import java.nio.file.*;
+            import java.io.IOException;
+            
+            public class $clazzName {
+                Path executionLogPath = Paths.get("test-executions" + getClass().getSimpleName() +".log");
+                
+                @Before 
+                public void beforeTest() {
+                    logExecution();
+                }
+                
+                @After 
+                public void afterTest() {
+                }
+                
+                @Test 
+                public void someTest() {
+                    ${content}
+                }
+                
+                int countExecutions() {
+                    try {
+                        return Files.readAllLines(executionLogPath).size();
+                    }
+                    catch(IOException e) {
+                        return 0;
+                    }
+                }
+               
+                void logExecution() {
+                    try {
+                        Files.write(executionLogPath, "Test executed\\n".getBytes(), StandardOpenOption.CREATE, StandardOpenOption.APPEND);
+                    } catch (IOException e) {
+                        // exception handling
+                    }
+                }
+            }
+        """
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunPlugin.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rerun;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.tasks.testing.Test;
+
+import javax.inject.Inject;
+
+import static org.elasticsearch.gradle.internal.test.rerun.TestTaskConfigurer.configureTestTask;
+
+public class TestRerunPlugin implements Plugin<Project> {
+
+    private final ObjectFactory objectFactory;
+
+    @Inject
+    TestRerunPlugin(ObjectFactory objectFactory) {
+        this.objectFactory = objectFactory;
+    }
+
+    @Override
+    public void apply(Project project) {
+        project.getTasks().withType(Test.class).configureEach(task -> configureTestTask(task, objectFactory));
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rerun;
+
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.testing.Test;
+
+import javax.inject.Inject;
+
+/**
+ * Allows configuring test rerun mechanics.
+ * <p>
+ * This extension is added with the name 'rerun' to all {@link Test} tasks.
+ */
+public class TestRerunTaskExtension {
+
+    /**
+     * The default number of reruns we allow for a test task.
+     */
+    public static final Integer DEFAULT_MAX_RERUNS = 1;
+
+    /**
+     * The name of the extension added to each test task.
+     */
+    public static String NAME = "rerun";
+
+    private final Property<Integer> maxReruns;
+
+    private final Property<Boolean> didRerun;
+
+    @Inject
+    public TestRerunTaskExtension(ObjectFactory objects) {
+        this.maxReruns = objects.property(Integer.class).convention(DEFAULT_MAX_RERUNS);
+        this.didRerun = objects.property(Boolean.class).convention(Boolean.FALSE);
+    }
+
+    /**
+     * The maximum number of times to rerun all tests.
+     * <p>
+     * This setting defaults to {@code 0}, which results in no retries.
+     * Any value less than 1 disables rerunning.
+     *
+     * @return the maximum number of times to rerun all tests of a task
+     */
+    public Property<Integer> getMaxReruns() {
+        return maxReruns;
+    }
+
+    /**
+     /**
+     * @return whether tests tests have been rerun or not. Defaults to false.
+     */
+    public Property<Boolean> getDidRerun() {
+        return didRerun;
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestTaskConfigurer.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rerun;
+
+import org.elasticsearch.gradle.internal.test.rerun.executer.RerunTestExecuter;
+import org.gradle.api.Action;
+import org.gradle.api.Task;
+import org.gradle.api.internal.tasks.testing.JvmTestExecutionSpec;
+import org.gradle.api.internal.tasks.testing.TestExecuter;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.tasks.testing.Test;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+public final class TestTaskConfigurer {
+
+    private TestTaskConfigurer() {}
+
+    public static void configureTestTask(Test test, ObjectFactory objectFactory) {
+        TestRerunTaskExtension extension = test.getExtensions()
+            .create(TestRerunTaskExtension.NAME, TestRerunTaskExtension.class, objectFactory);
+        test.doFirst(new InitTaskAction(extension));
+    }
+
+    private static RerunTestExecuter createRetryTestExecuter(Task task, TestRerunTaskExtension extension) {
+        TestExecuter<JvmTestExecutionSpec> delegate = getTestExecuter(task);
+        return new RerunTestExecuter(extension, delegate);
+    }
+
+    private static TestExecuter<JvmTestExecutionSpec> getTestExecuter(Task task) {
+        return invoke(declaredMethod(Test.class, "createTestExecuter"), task);
+    }
+
+    private static void setTestExecuter(Task task, RerunTestExecuter rerunTestExecuter) {
+        invoke(declaredMethod(Test.class, "setTestExecuter", TestExecuter.class), task, rerunTestExecuter);
+    }
+
+    private static class InitTaskAction implements Action<Task> {
+
+        private final TestRerunTaskExtension extension;
+
+        InitTaskAction(TestRerunTaskExtension extension) {
+            this.extension = extension;
+        }
+
+        @Override
+        public void execute(Task task) {
+            RerunTestExecuter retryTestExecuter = createRetryTestExecuter(task, extension);
+            setTestExecuter(task, retryTestExecuter);
+        }
+    }
+
+    private static Method declaredMethod(Class<?> type, String methodName, Class<?>... paramTypes) {
+        try {
+            return makeAccessible(type.getDeclaredMethod(methodName, paramTypes));
+        } catch (NoSuchMethodException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static Method makeAccessible(Method method) {
+        method.setAccessible(true);
+        return method;
+    }
+
+    private static <T> T invoke(Method method, Object instance, Object... args) {
+        try {
+            Object result = method.invoke(instance, args);
+            @SuppressWarnings("unchecked")
+            T cast = (T) result;
+            return cast;
+        } catch (IllegalAccessException | InvocationTargetException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestExecuter.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rerun.executer;
+
+import org.elasticsearch.gradle.internal.test.rerun.TestRerunTaskExtension;
+import org.gradle.api.GradleException;
+import org.gradle.api.internal.tasks.testing.JvmTestExecutionSpec;
+import org.gradle.api.internal.tasks.testing.TestDescriptorInternal;
+import org.gradle.api.internal.tasks.testing.TestExecuter;
+import org.gradle.api.internal.tasks.testing.TestResultProcessor;
+import org.gradle.internal.id.CompositeIdGenerator;
+import org.gradle.process.internal.ExecException;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public final class RerunTestExecuter implements TestExecuter<JvmTestExecutionSpec> {
+
+    private final TestRerunTaskExtension extension;
+    private final TestExecuter<JvmTestExecutionSpec> delegate;
+
+    public RerunTestExecuter(TestRerunTaskExtension extension, TestExecuter<JvmTestExecutionSpec> delegate) {
+        this.extension = extension;
+        this.delegate = delegate;
+    }
+
+    @Override
+    public void execute(JvmTestExecutionSpec spec, TestResultProcessor testResultProcessor) {
+        int maxRetries = extension.getMaxReruns().get();
+        if (maxRetries <= 0) {
+            delegate.execute(spec, testResultProcessor);
+            return;
+        }
+
+        RerunTestResultProcessor retryTestResultProcessor = new RerunTestResultProcessor(testResultProcessor);
+
+        int retryCount = 0;
+        JvmTestExecutionSpec testExecutionSpec = spec;
+        while (true) {
+            try {
+                delegate.execute(testExecutionSpec, retryTestResultProcessor);
+                break;
+            } catch (ExecException e) {
+                extension.getDidRerun().set(true);
+                report(retryCount + 1, retryTestResultProcessor.getActiveDescriptors());
+                if (retryCount++ == maxRetries) {
+                    throw new GradleException("Max retries(" + maxRetries + ") hit", e);
+                } else {
+                    retryTestResultProcessor.reset();
+                }
+            }
+        }
+    }
+
+    @Override
+    public void stopNow() {
+        delegate.stopNow();
+    }
+
+    void report(int runCount, List<TestDescriptorInternal> activeDescriptors) {
+        String report = "================\n"
+            + "Test jvm exited unexpectedly.\n"
+            + "Test jvm system exit trace (run: "
+            + runCount
+            + ")\n"
+            + activeDescriptors.stream()
+                .filter(d -> d.getId() instanceof CompositeIdGenerator.CompositeId)
+                .sorted(Comparator.comparing(o -> o.getId().toString()))
+                .map(TestDescriptorInternal::getName)
+                .collect(Collectors.joining(" > "))
+            + "\n"
+            + "================\n";
+        System.out.println(report);
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessor.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rerun.executer;
+
+import org.gradle.api.internal.tasks.testing.TestCompleteEvent;
+import org.gradle.api.internal.tasks.testing.TestDescriptorInternal;
+import org.gradle.api.internal.tasks.testing.TestResultProcessor;
+import org.gradle.api.internal.tasks.testing.TestStartEvent;
+import org.gradle.api.tasks.testing.TestOutputEvent;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+final class RerunTestResultProcessor implements TestResultProcessor {
+
+    private final TestResultProcessor delegate;
+
+    private final Map<Object, TestDescriptorInternal> activeDescriptorsById = new HashMap<>();
+
+    /**
+     * gradle structures tests in a tree structure with the test task itself
+     * being the root element. This is required to be tracked here to get the
+     * structure right when rerunning a test task tests.
+     * */
+    private Object rootTestDescriptorId;
+
+    RerunTestResultProcessor(TestResultProcessor delegate) {
+        this.delegate = delegate;
+    }
+
+    @Override
+    public void started(TestDescriptorInternal descriptor, TestStartEvent testStartEvent) {
+        if (rootTestDescriptorId == null) {
+            rootTestDescriptorId = descriptor.getId();
+            activeDescriptorsById.put(descriptor.getId(), descriptor);
+            delegate.started(descriptor, testStartEvent);
+        } else if (descriptor.getId().equals(rootTestDescriptorId) == false) {
+            activeDescriptorsById.put(descriptor.getId(), descriptor);
+            delegate.started(descriptor, testStartEvent);
+        }
+    }
+
+    @Override
+    public void completed(Object testId, TestCompleteEvent testCompleteEvent) {
+        if (testId.equals(rootTestDescriptorId)) {
+            if (activeDescriptorsById.size() != 1) {
+                return;
+            }
+        } else {
+            activeDescriptorsById.remove(testId);
+        }
+
+        delegate.completed(testId, testCompleteEvent);
+    }
+
+    @Override
+    public void output(Object testId, TestOutputEvent testOutputEvent) {
+        delegate.output(testId, testOutputEvent);
+    }
+
+    @Override
+    public void failure(Object testId, Throwable throwable) {
+        delegate.failure(testId, throwable);
+    }
+
+    public void reset() {
+        this.activeDescriptorsById.clear();
+    }
+
+    public List<TestDescriptorInternal> getActiveDescriptors() {
+        return new ArrayList<>(activeDescriptorsById.values());
+    }
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-test-rerun.properties

@@ -0,0 +1 @@
+implementation-class=org.elasticsearch.gradle.internal.test.rerun.TestRerunPlugin

gradle/build-scan.gradle

@@ -96,3 +96,19 @@ buildScan {
     }
   }
 }
+
+subprojects {
+  tasks.withType(Test).configureEach(new Action<Test>() {
+    @Override
+    void execute(Test test) {
+      test.doLast(new Action<Test>() {
+        @Override
+        void execute(Test t) {
+          if(t.rerun.didRerun.get() == true) {
+            buildScan.tag 'unexpected-test-jvm-exit'
+          }
+        }
+      })
+    }
+  })
+}