[Gradle/BWC] Patch bundled OpenJdk17 with Adoptium Jdk17 for older ES Distros (#135300) (#135360)

Bundled OpenJDK 17 is incompatible with newer versions of Ubuntu 24.04 (Kernel 6.14.x).
We fix our bwc testing on ubuntu 24.04 to explicitly use adoptium jdk 17 in cases where the bundled JDK is
older than 21 which is not affected.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -44,10 +44,14 @@ import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.internal.artifacts.dependencies.ProjectDependencyInternal;
+import org.gradle.api.plugins.JvmToolchainsPlugin;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.ClasspathNormalizer;
 import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.util.PatternFilterable;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaToolchainService;
+import org.gradle.jvm.toolchain.JvmVendorSpec;
 
 import java.util.Collection;
 import java.util.Iterator;
@@ -59,6 +63,7 @@ import java.util.Optional;
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
+import static org.elasticsearch.gradle.util.OsUtils.jdkIsIncompatibleWithOS;
 
 /**
  * Base plugin used for wiring up build tasks to REST testing tasks using new JUnit rule-based test clusters framework.
@@ -93,6 +98,7 @@ public class RestTestBasePlugin implements Plugin<Project> {
     public void apply(Project project) {
         project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
         project.getPluginManager().apply(InternalDistributionDownloadPlugin.class);
+        project.getPluginManager().apply(JvmToolchainsPlugin.class);
         var bwcVersions = loadBuildParams(project).get().getBwcVersions();
 
         // Register integ-test and default distributions
@@ -226,6 +232,17 @@ public class RestTestBasePlugin implements Plugin<Project> {
                     String versionString = version.toString();
                     ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
 
+                    if (jdkIsIncompatibleWithOS(Version.fromString(versionString))) {
+                        var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
+                        var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
+                            spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
+                            spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
+                        });
+                        task.environment(
+                            "ES_FALLBACK_JAVA_HOME",
+                            fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
+                        );
+                    }
                     task.dependsOn(bwcDistro);
                     registerDistributionInputs(task, bwcDistro);
 

build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy

@@ -9,14 +9,14 @@
 
 package org.elasticsearch.gradle
 
-import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
-import org.gradle.testkit.runner.GradleRunner
 import spock.lang.IgnoreIf
 import spock.lang.Unroll
+import spock.util.environment.RestoreSystemProperties
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.GradleRunner
 
-import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withChangedClasspathMockedDistributionDownload
-import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withChangedConfigMockedDistributionDownload
-import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withMockedDistributionDownload
+import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.*
 
 /**
  * We do not have coverage for the test cluster startup on windows yet.
@@ -108,8 +108,8 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
         def runningClosure = { GradleRunner r -> r.build() }
         withMockedDistributionDownload(runner, runningClosure)
         def result = inputProperty == "distributionClasspath" ?
-                withChangedClasspathMockedDistributionDownload(runner, runningClosure) :
-                withChangedConfigMockedDistributionDownload(runner, runningClosure)
+            withChangedClasspathMockedDistributionDownload(runner, runningClosure) :
+            withChangedConfigMockedDistributionDownload(runner, runningClosure)
 
         then:
         result.output.contains("Task ':myTask' is not up-to-date because:\n  Input property 'clusters.myCluster\$0.nodes.\$0.$inputProperty'")
@@ -166,18 +166,24 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
         }
 
         then:
-        result.output.contains("Task ':myTask' is not up-to-date because:\n" +
-                "  Input property 'clusters.myCluster\$0.$propertyName'")
+        result.output.contains(
+            "Task ':myTask' is not up-to-date because:\n" +
+                "  Input property 'clusters.myCluster\$0.$propertyName'"
+        )
         result.output.contains("elasticsearch-keystore script executed!")
         assertEsOutputContains("myCluster", "Starting Elasticsearch process")
         assertEsOutputContains("myCluster", "Stopping node")
 
         where:
         pluginType | propertyName         | fileChange
-        'module'   | "installedFiles"     | { def testClazz -> testClazz.file("test-module/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
-        'plugin'   | "installedFiles"     | { def testClazz -> testClazz.file("test-plugin/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
-        'module'   | "installedClasspath" | { def testClazz -> testClazz.file("test-module/src/main/java/SomeClass.java") << "class SomeClass {}" }
-        'plugin'   | "installedClasspath" | { def testClazz -> testClazz.file("test-plugin/src/main/java/SomeClass.java") << "class SomeClass {}" }
+        'module'   | "installedFiles"     |
+            { def testClazz -> testClazz.file("test-module/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
+        'plugin'   | "installedFiles"     |
+            { def testClazz -> testClazz.file("test-plugin/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
+        'module'   | "installedClasspath" |
+            { def testClazz -> testClazz.file("test-module/src/main/java/SomeClass.java") << "class SomeClass {}" }
+        'plugin'   | "installedClasspath" |
+            { def testClazz -> testClazz.file("test-plugin/src/main/java/SomeClass.java") << "class SomeClass {}" }
     }
 
     def "can declare test cluster in lazy evaluated task configuration block"() {
@@ -232,9 +238,51 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
         assertCustomDistro('myCluster')
     }
 
+    @RestoreSystemProperties
+    def "override jdk usage via ES_JAVA_HOME for known jdk os incompatibilities"() {
+        given:
+
+        settingsFile.text = """
+            plugins {
+                id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0'
+            }
+            """ + settingsFile.text
+
+        buildFile << """
+            testClusters {
+              myCluster {
+                testDistribution = 'default'
+                version = '8.10.4'
+              }
+            }
+
+            // Force linux platform to trigger jdk override
+            elasticsearch_distributions.forEach { d ->
+                d.platform = org.elasticsearch.gradle.ElasticsearchDistribution.Platform.LINUX
+            }
+
+            tasks.register('myTask', SomeClusterAwareTask) {
+                useCluster testClusters.myCluster
+            }
+        """
+        when:
+        def result = withMockedDistributionDownload(
+            "8.10.4",
+            ElasticsearchDistribution.Platform.LINUX,
+            gradleRunner("myTask", '-Dos.name=Linux', '-Dos.version=6.14.0-1015-gcp', '-i')
+        ) {
+            build()
+        }
+
+        then:
+        result.output.lines().anyMatch { line -> line.startsWith("Running") && line.split().find { it.startsWith("ES_JAVA_HOME=") }.contains("eclipse_adoptium-17") }
+    }
+
     boolean assertEsOutputContains(String testCluster, String expectedOutput) {
-        assert new File(testProjectDir.root,
-                "build/testclusters/${testCluster}-0/logs/es.out").text.contains(expectedOutput)
+        assert new File(
+            testProjectDir.root,
+            "build/testclusters/${testCluster}-0/logs/es.out"
+        ).text.contains(expectedOutput)
         true
     }
 

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -37,6 +37,7 @@ import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.bundling.AbstractArchiveTask;
 import org.gradle.api.tasks.bundling.Zip;
+import org.gradle.jvm.toolchain.JavaLauncher;
 import org.gradle.process.ExecOperations;
 
 import java.io.File;
@@ -84,6 +85,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
     private int nodeIndex = 0;
 
     private final ConfigurableFileCollection pluginAndModuleConfiguration;
+    private final Provider<JavaLauncher> jdk17FallbackLauncher;
 
     private boolean shared = false;
 
@@ -101,7 +103,8 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
         FileOperations fileOperations,
         File workingDirBase,
         Provider<File> runtimeJava,
-        Function<Version, Boolean> isReleasedVersion
+        Function<Version, Boolean> isReleasedVersion,
+        Provider<JavaLauncher> jdk17FallbackLauncher
     ) {
         this.path = path;
         this.clusterName = clusterName;
@@ -117,6 +120,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
         this.isReleasedVersion = isReleasedVersion;
         this.nodes = project.container(ElasticsearchNode.class);
         this.pluginAndModuleConfiguration = project.getObjects().fileCollection();
+        this.jdk17FallbackLauncher = jdk17FallbackLauncher;
         this.nodes.add(
             new ElasticsearchNode(
                 safeName(clusterName),
@@ -131,7 +135,8 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
                 fileOperations,
                 workingDirBase,
                 runtimeJava,
-                isReleasedVersion
+                isReleasedVersion,
+                jdk17FallbackLauncher
             )
         );
 
@@ -189,7 +194,8 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
                     fileOperations,
                     workingDirBase,
                     runtimeJava,
-                    isReleasedVersion
+                    isReleasedVersion,
+                    jdk17FallbackLauncher
                 )
             );
         }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -49,6 +49,7 @@ import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.bundling.Zip;
 import org.gradle.api.tasks.util.PatternFilterable;
+import org.gradle.jvm.toolchain.JavaLauncher;
 import org.gradle.process.ExecOperations;
 
 import java.io.ByteArrayInputStream;
@@ -94,6 +95,7 @@ import java.util.stream.Stream;
 
 import static java.util.Objects.requireNonNull;
 import static java.util.Optional.ofNullable;
+import static org.elasticsearch.gradle.util.OsUtils.jdkIsIncompatibleWithOS;
 
 public class ElasticsearchNode implements TestClusterConfiguration {
 
@@ -166,6 +168,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final Path tmpDir;
     private final Provider<File> runtimeJava;
     private final Function<Version, Boolean> isReleasedVersion;
+    private final Provider<JavaLauncher> jdk17FallbackLauncher;
     private final List<ElasticsearchDistribution> distributions = new ArrayList<>();
     private int currentDistro = 0;
     private TestDistribution testDistribution;
@@ -190,7 +193,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         FileOperations fileOperations,
         File workingDirBase,
         Provider<File> runtimeJava,
-        Function<Version, Boolean> isReleasedVersion
+        Function<Version, Boolean> isReleasedVersion,
+        Provider<JavaLauncher> jdk17FallbackLauncher
     ) {
         this.path = path;
         this.name = name;
@@ -203,6 +207,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         this.fileOperations = fileOperations;
         this.runtimeJava = runtimeJava;
         this.isReleasedVersion = isReleasedVersion;
+        this.jdk17FallbackLauncher = jdk17FallbackLauncher;
         workingDir = workingDirBase.toPath().resolve(safeName(name)).toAbsolutePath();
         confPathRepo = workingDir.resolve("repo");
         configFile = workingDir.resolve("config/elasticsearch.yml");
@@ -793,7 +798,19 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         if (getTestDistribution() == TestDistribution.INTEG_TEST || getVersion().equals(VersionProperties.getElasticsearchVersion())) {
             defaultEnv.put("ES_JAVA_HOME", runtimeJava.get().getAbsolutePath());
         }
+        // Older distributions ship with openjdk versions that are not compatible with newer kernels of ubuntu 24.04 and later
+        // Therefore we pass explicitly the runtime java to use the adoptium jdk that is maintained longer and compatible
+        // with newer kernels.
+        // 8.10.4 is the last version shipped with jdk < 21. We configure these cluster to run with jdk 17 adoptium as 17 was
+        // the last LTS release before 21
+        else if (jdkIsIncompatibleWithOS(getVersion())) {
+            defaultEnv.put(
+                "ES_JAVA_HOME",
+                jdk17FallbackLauncher.map(j -> j.getMetadata().getInstallationPath().getAsFile().getAbsolutePath()).get()
+            );
+        }
         defaultEnv.put("ES_PATH_CONF", configFile.getParent().toString());
+
         String systemPropertiesString = "";
         if (systemProperties.isEmpty() == false) {
             systemPropertiesString = " " + systemProperties.entrySet().stream().peek(entry -> {

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -26,6 +26,7 @@ import org.gradle.api.internal.file.FileOperations;
 import org.gradle.api.invocation.Gradle;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
+import org.gradle.api.plugins.JvmToolchainsPlugin;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
@@ -33,6 +34,10 @@ import org.gradle.api.services.BuildService;
 import org.gradle.api.services.BuildServiceParameters;
 import org.gradle.build.event.BuildEventsListenerRegistry;
 import org.gradle.internal.jvm.Jvm;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaLauncher;
+import org.gradle.jvm.toolchain.JavaToolchainService;
+import org.gradle.jvm.toolchain.JvmVendorSpec;
 import org.gradle.process.ExecOperations;
 import org.gradle.tooling.events.FinishEvent;
 import org.gradle.tooling.events.OperationCompletionListener;
@@ -99,11 +104,19 @@ public class TestClustersPlugin implements Plugin<Project> {
     @Override
     public void apply(Project project) {
         project.getPlugins().apply(DistributionDownloadPlugin.class);
+        project.getPlugins().apply(JvmToolchainsPlugin.class);
         project.getRootProject().getPluginManager().apply(ReaperPlugin.class);
         Provider<ReaperService> reaperServiceProvider = GradleUtils.getBuildService(
             project.getGradle().getSharedServices(),
             ReaperPlugin.REAPER_SERVICE_NAME
         );
+
+        JavaToolchainService toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
+        Provider<JavaLauncher> fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
+            spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
+            spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
+        });
+
         runtimeJavaProvider = providerFactory.provider(
             () -> System.getenv("RUNTIME_JAVA_HOME") == null ? Jvm.current().getJavaHome() : new File(System.getenv("RUNTIME_JAVA_HOME"))
         );
@@ -117,7 +130,8 @@ public class TestClustersPlugin implements Plugin<Project> {
         NamedDomainObjectContainer<ElasticsearchCluster> container = createTestClustersContainerExtension(
             project,
             testClustersRegistryProvider,
-            reaperServiceProvider
+            reaperServiceProvider,
+            fallbackJdk17Launcher
         );
 
         // provide a task to be able to list defined clusters.
@@ -154,7 +168,8 @@ public class TestClustersPlugin implements Plugin<Project> {
     private NamedDomainObjectContainer<ElasticsearchCluster> createTestClustersContainerExtension(
         Project project,
         Provider<TestClustersRegistry> testClustersRegistryProvider,
-        Provider<ReaperService> reaper
+        Provider<ReaperService> reaper,
+        Provider<JavaLauncher> fallbackJdk17Launcher
     ) {
         // Create an extensions that allows describing clusters
         NamedDomainObjectContainer<ElasticsearchCluster> container = project.container(
@@ -171,7 +186,8 @@ public class TestClustersPlugin implements Plugin<Project> {
                 getFileOperations(),
                 new File(project.getBuildDir(), "testclusters"),
                 runtimeJavaProvider,
-                isReleasedVersion
+                isReleasedVersion,
+                fallbackJdk17Launcher
             )
         );
         project.getExtensions().add(EXTENSION_NAME, container);

build-tools/src/main/java/org/elasticsearch/gradle/util/OsUtils.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.util;
+
+import org.elasticsearch.gradle.OS;
+import org.elasticsearch.gradle.Version;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public final class OsUtils {
+
+    private static final Logger LOGGER = Logging.getLogger(OsUtils.class);
+
+    private OsUtils() {}
+
+    /**
+      * OpenJDK 17 that we ship with for older ES distributions is incompatible with Ubuntu 24.04 and newer due to
+      * a change in newer kernel versions that causes JVM crashes.
+      * <p>
+      * See <a href="https://github.com/oracle/graal/issues/4831">https://github.com/oracle/graal/issues/4831</a> that exposes a similar issue with GraalVM.
+      * <p>
+      * It can be reproduced using Jshell on Ubuntu 24.04+ with:
+      * <pre>
+      * jshell&gt; java.lang.management.ManagementFactory.getOperatingSystemMXBean()
+      * |  Exception java.lang.NullPointerException: Cannot invoke "jdk.internal.platform.CgroupInfo.getMountPoint()" because "anyController" is null
+      * </pre>
+      * <p>
+      * This method returns true if the given version of the JDK is known to be incompatible
+      */
+    public static boolean jdkIsIncompatibleWithOS(Version version) {
+        return version.onOrBefore("8.10.4") && isUbuntu2404OrLater();
+    }
+
+    private static boolean isUbuntu2404OrLater() {
+        try {
+            if (OS.current() != OS.LINUX) {
+                return false;
+            }
+
+            // try reading kernel info from System properties first to make this better testable
+            String osKernelString = System.getProperty("os.version");
+            Version kernelVersion = Version.fromString(osKernelString, Version.Mode.RELAXED);
+            if (kernelVersion.onOrAfter("6.14.0")) {
+                return true;
+            }
+
+            // Read /etc/os-release file to get distribution info
+            Path osRelease = Path.of("/etc/os-release");
+            if (Files.exists(osRelease) == false) {
+                return false;
+            }
+
+            String content = Files.readString(osRelease);
+            boolean isUbuntu = content.contains("ID=ubuntu");
+
+            if (isUbuntu == false) {
+                return false;
+            }
+
+            // Extract version
+            String versionLine = content.lines().filter(line -> line.startsWith("VERSION_ID=")).findFirst().orElse("");
+
+            if (versionLine.isEmpty()) {
+                return false;
+            }
+
+            String version = versionLine.substring("VERSION_ID=".length()).replace("\"", "");
+            String[] parts = version.split("\\.");
+
+            if (parts.length >= 2) {
+                int major = Integer.parseInt(parts[0]);
+                int minor = Integer.parseInt(parts[1]);
+                return major > 24 || (major == 24 && minor >= 4);
+            }
+
+            return false;
+        } catch (Exception e) {
+            LOGGER.debug("Failed to detect Ubuntu version", e);
+            return false;
+        }
+    }
+
+}

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java

@@ -82,6 +82,7 @@ public abstract class AbstractLocalClusterFactory<S extends LocalClusterSpec, H
     private static final String ENABLE_DEBUG_JVM_ARGS = "-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=";
     private static final String ENTITLEMENT_POLICY_YAML = "entitlement-policy.yaml";
     private static final String PLUGIN_DESCRIPTOR_PROPERTIES = "plugin-descriptor.properties";
+    public static final String DISTRO_WITH_JDK_LOWER_21 = "8.11.0";
 
     private final DistributionResolver distributionResolver;
 
@@ -867,6 +868,10 @@ public abstract class AbstractLocalClusterFactory<S extends LocalClusterSpec, H
 
         private Map<String, String> getEnvironmentVariables() {
             Map<String, String> environment = new HashMap<>(spec.resolveEnvironment());
+            String esFallbackJavaHome = System.getenv("ES_FALLBACK_JAVA_HOME");
+            if (spec.getVersion().before(DISTRO_WITH_JDK_LOWER_21) && esFallbackJavaHome != null && esFallbackJavaHome.isEmpty() == false) {
+                environment.put("ES_JAVA_HOME", esFallbackJavaHome);
+            }
             environment.put("ES_PATH_CONF", configDir.toString());
             environment.put("ES_TMPDIR", workingDir.resolve("tmp").toString());
             // Windows requires this as it defaults to `c:\windows` despite ES_TMPDIR