Introduce aarch64 packaging (#53914)

This commit introduces aarch64 packaging, including bundling an aarch64
JDK distribution. We had to make some interesting choices here:
 - ML binaries are not compiled for aarch64, so for now we disable ML on
   aarch64
 - depending on underlying page sizes, we have to disable class data
   sharing

buildSrc/build.gradle

@@ -187,8 +187,10 @@ if (project != rootProject) {
     distribution project(':distribution:archives:oss-windows-zip')
     distribution project(':distribution:archives:darwin-tar')
     distribution project(':distribution:archives:oss-darwin-tar')
+    distribution project(':distribution:archives:linux-aarch64-tar')
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:oss-linux-tar')
+    distribution project(':distribution:archives:oss-linux-aarch64-tar')
   }
 
   // for external projects we want to remove the marker file indicating we are running the Elasticsearch project

buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+public enum Architecture {
+
+    X64,
+    AARCH64;
+
+    public static Architecture current() {
+        final String architecture = System.getProperty("os.arch", "");
+        switch (architecture) {
+            case "amd64":
+            case "x86_64":
+                return X64;
+            case "aarch64":
+                return AARCH64;
+            default:
+                throw new IllegalArgumentException("can not determine architecture from [" + architecture + "]");
+        }
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java

@@ -33,6 +33,7 @@ import java.util.regex.Pattern;
 
 public class Jdk implements Buildable, Iterable<File> {
 
+    private static final List<String> ALLOWED_ARCHITECTURES = List.of("aarch64", "x64");
     private static final List<String> ALLOWED_VENDORS = List.of("adoptopenjdk", "openjdk");
     private static final List<String> ALLOWED_PLATFORMS = List.of("darwin", "linux", "windows", "mac");
     private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
@@ -44,6 +45,7 @@ public class Jdk implements Buildable, Iterable<File> {
     private final Property<String> vendor;
     private final Property<String> version;
     private final Property<String> platform;
+    private final Property<String> architecture;
     private String baseVersion;
     private String major;
     private String build;
@@ -55,6 +57,7 @@ public class Jdk implements Buildable, Iterable<File> {
         this.vendor = objectFactory.property(String.class);
         this.version = objectFactory.property(String.class);
         this.platform = objectFactory.property(String.class);
+        this.architecture = objectFactory.property(String.class);
     }
 
     public String getName() {
@@ -97,6 +100,19 @@ public class Jdk implements Buildable, Iterable<File> {
         this.platform.set(platform);
     }
 
+    public String getArchitecture() {
+        return architecture.get();
+    }
+
+    public void setArchitecture(final String architecture) {
+        if (ALLOWED_ARCHITECTURES.contains(architecture) == false) {
+            throw new IllegalArgumentException(
+                "unknown architecture [" + architecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES
+            );
+        }
+        this.architecture.set(architecture);
+    }
+
     public String getBaseVersion() {
         return baseVersion;
     }
@@ -153,9 +169,13 @@ public class Jdk implements Buildable, Iterable<File> {
         if (vendor.isPresent() == false) {
             throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]");
         }
+        if (architecture.isPresent() == false) {
+            throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]");
+        }
         version.finalizeValue();
         platform.finalizeValue();
         vendor.finalizeValue();
+        architecture.finalizeValue();
     }
 
     @Override

buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

@@ -47,7 +47,6 @@ import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.stream.StreamSupport;
 
-import static org.elasticsearch.gradle.Util.capitalize;
 import static org.elasticsearch.gradle.tool.Boilerplate.findByName;
 import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate;
 
@@ -72,7 +71,10 @@ public class JdkDownloadPlugin implements Plugin<Project> {
                 DependencyHandler dependencies = project.getDependencies();
                 Map<String, Object> depConfig = new HashMap<>();
                 depConfig.put("path", ":"); // root project
-                depConfig.put("configuration", configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform()));
+                depConfig.put(
+                    "configuration",
+                    configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture())
+                );
                 project.getDependencies().add(jdk.getConfigurationName(), dependencies.project(depConfig));
 
                 // ensure a root level jdk download task exists
@@ -87,7 +89,14 @@ public class JdkDownloadPlugin implements Plugin<Project> {
     }
 
     private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
-        String extractTaskName = "extract" + capitalize(jdk.getPlatform()) + "Jdk-" + jdk.getVendor() + "-" + jdk.getVersion();
+        String extractTaskName = String.format(
+            Locale.ROOT,
+            "extract-%s-%s-jdk-%s-%s",
+            jdk.getPlatform(),
+            jdk.getArchitecture(),
+            jdk.getVendor(),
+            jdk.getVersion()
+        );
 
         // Skip setup if we've already configured a JDK for this platform, vendor and version
         if (findByName(rootProject.getTasks(), extractTaskName) == null) {
@@ -107,7 +116,7 @@ public class JdkDownloadPlugin implements Plugin<Project> {
                 repoUrl = "https://artifactory.elstc.co/artifactory/oss-jdk-local/";
                 artifactPattern = String.format(
                     Locale.ROOT,
-                    "adoptopenjdk/OpenJDK%sU-jdk_x64_[module]_hotspot_[revision]_%s.[ext]",
+                    "adoptopenjdk/OpenJDK%sU-jdk_[classifier]_[module]_hotspot_[revision]_%s.[ext]",
                     jdk.getMajor(),
                     jdk.getBuild()
                 );
@@ -121,14 +130,14 @@ public class JdkDownloadPlugin implements Plugin<Project> {
                         + jdk.getHash()
                         + "/"
                         + jdk.getBuild()
-                        + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
                 } else {
                     // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
                     artifactPattern = "java/GA/jdk"
                         + jdk.getMajor()
                         + "/"
                         + jdk.getBuild()
-                        + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
                 }
             } else {
                 throw new GradleException("Unknown JDK vendor [" + jdk.getVendor() + "]");
@@ -150,14 +159,14 @@ public class JdkDownloadPlugin implements Plugin<Project> {
 
             // Declare a configuration and dependency from which to download the remote JDK
             final ConfigurationContainer configurations = rootProject.getConfigurations();
-            String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
+            String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture());
             Configuration downloadConfiguration = maybeCreate(configurations, downloadConfigName);
             rootProject.getDependencies().add(downloadConfigName, dependencyNotation(jdk));
 
             // Create JDK extract task
             final Provider<Directory> extractPath = rootProject.getLayout()
                 .getBuildDirectory()
-                .dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform());
+                .dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform() + "_" + jdk.getArchitecture());
 
             TaskProvider<?> extractTask = createExtractTask(
                 extractTaskName,
@@ -168,7 +177,13 @@ public class JdkDownloadPlugin implements Plugin<Project> {
             );
 
             // Declare a configuration for the extracted JDK archive
-            String artifactConfigName = configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
+            String artifactConfigName = configName(
+                "extracted_jdk",
+                jdk.getVendor(),
+                jdk.getVersion(),
+                jdk.getPlatform(),
+                jdk.getArchitecture()
+            );
             maybeCreate(configurations, artifactConfigName);
             rootProject.getArtifacts().add(artifactConfigName, extractPath, artifact -> artifact.builtBy(extractTask));
         }
@@ -254,7 +269,7 @@ public class JdkDownloadPlugin implements Plugin<Project> {
             : jdk.getPlatform();
         String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
 
-        return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
+        return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + ":" + jdk.getArchitecture() + "@" + extension;
     }
 
     private static String groupName(Jdk jdk) {

buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -175,12 +175,14 @@ public class DistroTestPlugin implements Plugin<Project> {
         String name,
         String vendor,
         String version,
-        String platform
+        String platform,
+        String architecture
     ) {
         Jdk jdk = jdksContainer.create(name);
         jdk.setVendor(vendor);
         jdk.setVersion(version);
         jdk.setPlatform(platform);
+        jdk.setArchitecture(architecture);
         return jdk;
     }
 
@@ -212,8 +214,8 @@ public class DistroTestPlugin implements Plugin<Project> {
 
         NamedDomainObjectContainer<Jdk> jdksContainer = JdkDownloadPlugin.getContainer(project);
         String platform = box.contains("windows") ? "windows" : "linux";
-        Jdk systemJdk = createJdk(jdksContainer, "system", SYSTEM_JDK_VENDOR, SYSTEM_JDK_VERSION, platform);
-        Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform);
+        Jdk systemJdk = createJdk(jdksContainer, "system", SYSTEM_JDK_VENDOR, SYSTEM_JDK_VERSION, platform, "x64");
+        Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform, "x64");
 
         // setup VM used by these tests
         VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class);

buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java

@@ -36,7 +36,7 @@ public class JdkDownloadPluginTests extends GradleUnitTestCase {
     }
 
     public void testMissingVendor() {
-        assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "vendor not specified for jdk [testjdk]");
+        assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "x64", "vendor not specified for jdk [testjdk]");
     }
 
     public void testUnknownVendor() {
@@ -46,20 +46,29 @@ public class JdkDownloadPluginTests extends GradleUnitTestCase {
             "unknown",
             "11.0.2+33",
             "linux",
+            "x64",
             "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]"
         );
     }
 
     public void testMissingVersion() {
-        assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "version not specified for jdk [testjdk]");
+        assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "x64", "version not specified for jdk [testjdk]");
     }
 
     public void testBadVersionFormat() {
-        assertJdkError(createProject(), "testjdk", "openjdk", "badversion", "linux", "malformed version [badversion] for jdk [testjdk]");
+        assertJdkError(
+            createProject(),
+            "testjdk",
+            "openjdk",
+            "badversion",
+            "linux",
+            "x64",
+            "malformed version [badversion] for jdk [testjdk]"
+        );
     }
 
     public void testMissingPlatform() {
-        assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "platform not specified for jdk [testjdk]");
+        assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "x64", "platform not specified for jdk [testjdk]");
     }
 
     public void testUnknownPlatform() {
@@ -69,19 +78,44 @@ public class JdkDownloadPluginTests extends GradleUnitTestCase {
             "openjdk",
             "11.0.2+33",
             "unknown",
+            "x64",
             "unknown platform [unknown] for jdk [testjdk], must be one of [darwin, linux, windows, mac]"
         );
     }
 
-    private void assertJdkError(Project project, String name, String vendor, String version, String platform, String message) {
+    public void testMissingArchitecture() {
+        assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", "linux", null, "architecture not specified for jdk [testjdk]");
+    }
+
+    public void testUnknownArchitecture() {
+        assertJdkError(
+            createProject(),
+            "testjdk",
+            "openjdk",
+            "11.0.2+33",
+            "linux",
+            "unknown",
+            "unknown architecture [unknown] for jdk [testjdk], must be one of [aarch64, x64]"
+        );
+    }
+
+    private void assertJdkError(
+        final Project project,
+        final String name,
+        final String vendor,
+        final String version,
+        final String platform,
+        final String architecture,
+        final String message
+    ) {
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> createJdk(project, name, vendor, version, platform)
+            () -> createJdk(project, name, vendor, version, platform, architecture)
         );
         assertThat(e.getMessage(), equalTo(message));
     }
 
-    private void createJdk(Project project, String name, String vendor, String version, String platform) {
+    private void createJdk(Project project, String name, String vendor, String version, String platform, String architecture) {
         @SuppressWarnings("unchecked")
         NamedDomainObjectContainer<Jdk> jdks = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
         jdks.create(name, jdk -> {
@@ -94,6 +128,9 @@ public class JdkDownloadPluginTests extends GradleUnitTestCase {
             if (platform != null) {
                 jdk.setPlatform(platform);
             }
+            if (architecture != null) {
+                jdk.setArchitecture(architecture);
+            }
         }).finalizeValues();
     }
 

buildSrc/src/testKit/jdk-download/reuse/build.gradle

@@ -7,5 +7,6 @@ jdks {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "linux"
+    architeecture = "x64"
   }
 }

buildSrc/src/testKit/jdk-download/subproj/build.gradle

@@ -10,16 +10,19 @@ jdks {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "linux"
+    architecture = "x64"
   }
   darwin {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "darwin"
+    architecture = "x64"
   }
   windows {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "windows"
+    architecture = "x64"
   }
 }
 

distribution/archives/build.gradle

@@ -50,7 +50,7 @@ task createJvmOptionsDir(type: EmptyDirTask) {
   dirMode = 0750
 }
 
-CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) {
+CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, boolean oss, boolean jdk) {
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
@@ -70,7 +70,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
       }
       if (jdk) {
         into("darwin".equals(platform) ? 'jdk.app' : 'jdk') {
-          with jdkFiles(project, platform)
+          with jdkFiles(project, platform, architecture)
         }
       }
       into('') {
@@ -116,31 +116,31 @@ Closure commonZipConfig = {
 
 task buildIntegTestZip(type: Zip) {
   configure(commonZipConfig)
-  with archiveFiles(transportModulesFiles, 'zip', null, true, false)
+  with archiveFiles(transportModulesFiles, 'zip', null, 'x64', true, false)
 }
 
 task buildWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', false, true)
+  with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', 'x64', false, true)
 }
 
 task buildOssWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', true, true)
+  with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', 'x64', true, true)
 }
 
 task buildNoJdkWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'no-jdk-windows-x86_64'
-  with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', false, false)
+  with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', 'x64', false, false)
 }
 
 task buildOssNoJdkWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'no-jdk-windows-x86_64'
-  with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', true, false)
+  with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', 'x64', true, false)
 }
 
 Closure commonTarConfig = {
@@ -153,49 +153,61 @@ Closure commonTarConfig = {
 task buildDarwinTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', false, true)
+  with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', 'x64', false, true)
 }
 
 task buildOssDarwinTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', true, true)
+  with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', 'x64', true, true)
 }
 
 task buildNoJdkDarwinTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-darwin-x86_64'
-  with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', false, false)
+  with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', 'x64', false, false)
 }
 
 task buildOssNoJdkDarwinTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-darwin-x86_64'
-  with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', true, false)
+  with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', 'x64', true, false)
+}
+
+task buildLinuxAarch64Tar(type: SymbolicLinkPreservingTar) {
+  configure(commonTarConfig)
+  archiveClassifier = 'linux-aarch64'
+  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'aarch64', false, true)
 }
 
 task buildLinuxTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', false, true)
+  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'x64', false, true)
+}
+
+task buildOssLinuxAarch64Tar(type: SymbolicLinkPreservingTar) {
+  configure(commonTarConfig)
+  archiveClassifier = 'linux-x86_64'
+  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, true)
 }
 
 task buildOssLinuxTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', true, true)
+  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, true)
 }
 
 task buildNoJdkLinuxTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-linux-x86_64'
-  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', false, false)
+  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'x64', false, false)
 }
 
 task buildOssNoJdkLinuxTar(type: SymbolicLinkPreservingTar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-linux-x86_64'
-  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', true, false)
+  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, false)
 }
 
 Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }

distribution/archives/linux-aarch64-tar/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/archives/oss-linux-aarch64-tar/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/build.gradle

@@ -391,16 +391,17 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       }
     }
 
-    jdkFiles = { Project project, String platform ->
+    jdkFiles = { Project project, String platform, String architecture ->
       project.jdks {
-        "bundled_${platform}" {
+        "bundled_${platform}_${architecture}" {
           it.platform = platform
           it.version = VersionProperties.getBundledJdk(platform)
           it.vendor = VersionProperties.bundledJdkVendor
+          it.architecture = architecture
         }
       }
       return copySpec {
-        from project.jdks."bundled_${platform}"
+        from project.jdks."bundled_${platform}_${architecture}"
         exclude "demo/**"
         eachFile { FileCopyDetails details ->
           if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
@@ -607,10 +608,13 @@ subprojects {
 
 ['archives:windows-zip', 'archives:oss-windows-zip',
   'archives:darwin-tar', 'archives:oss-darwin-tar',
+  'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar',
   'archives:linux-tar', 'archives:oss-linux-tar',
   'archives:integ-test-zip',
   'packages:rpm', 'packages:deb',
+  'packages:aarch64-rpm', 'packages:aarch64-deb',
   'packages:oss-rpm', 'packages:oss-deb',
+  'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb'
 ].forEach { subName ->
   Project subproject = project("${project.path}:${subName}")
   Configuration configuration = configurations.create(subproject.name)

distribution/packages/aarch64-deb/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/packages/aarch64-oss-deb/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/packages/aarch64-oss-rpm/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/packages/aarch64-rpm/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/packages/build.gradle

@@ -98,17 +98,17 @@ addProcessFilesTask('rpm', false, false)
 // Common configuration that is package dependent. This can't go in ospackage
 // since we have different templated files that need to be consumed, but the structure
 // is the same
-Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
+Closure commonPackageConfig(String type, boolean oss, boolean jdk, String architecture) {
   return {
     onlyIf {
       OS.current().equals(OS.WINDOWS) == false
     }
     dependsOn "process${oss ? 'Oss' : ''}${jdk ? '' : 'NoJdk'}${type.capitalize()}Files"
     packageName "elasticsearch${oss ? '-oss' : ''}"
-    arch(type == 'deb' ? 'amd64' : 'X86_64')
+    arch(architecture == 'aarch64' ? 'aarch64' : type == 'deb' ? 'amd64' : 'X86_64')
     // Follow elasticsearch's file naming convention
     String jdkString = jdk ? "" : "no-jdk-"
-    String prefix = "${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
+    String prefix = "${architecture == 'aarch64' ? 'aarch64-' : ''}${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
     destinationDir = file("${prefix}/build/distributions")
 
     // SystemPackagingTask overrides default archive task convention mappings, but doesn't provide a setter so we have to override the convention mapping itself
@@ -143,7 +143,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
       }
       if (jdk) {
         into('jdk') {
-          with jdkFiles(project, 'linux')
+          with jdkFiles(project, 'linux', architecture)
         }
       }
       // we need to specify every intermediate directory in these paths so the package managers know they are explicitly
@@ -306,9 +306,9 @@ ospackage {
   into '/usr/share/elasticsearch'
 }
 
-Closure commonDebConfig(boolean oss, boolean jdk) {
+Closure commonDebConfig(boolean oss, boolean jdk, String architecture) {
   return {
-    configure(commonPackageConfig('deb', oss, jdk))
+    configure(commonPackageConfig('deb', oss, jdk, architecture))
 
     // jdeb does not provide a way to set the License control attribute, and ospackage
     // silently ignores setting it. Instead, we set the license as "custom field"
@@ -336,25 +336,33 @@ Closure commonDebConfig(boolean oss, boolean jdk) {
   }
 }
 
+task buildAarch64Deb(type: Deb) {
+  configure(commonDebConfig(false, true, 'aarch64'))
+}
+
 task buildDeb(type: Deb) {
-  configure(commonDebConfig(false, true))
+  configure(commonDebConfig(false, true, 'x64'))
+}
+
+task buildAarch64OssDeb(type: Deb) {
+  configure(commonDebConfig(true, true, 'aarch64'))
 }
 
 task buildOssDeb(type: Deb) {
-  configure(commonDebConfig(true, true))
+  configure(commonDebConfig(true, true, 'x64'))
 }
 
 task buildNoJdkDeb(type: Deb) {
-  configure(commonDebConfig(false, false))
+  configure(commonDebConfig(false, false, 'x64'))
 }
 
 task buildOssNoJdkDeb(type: Deb) {
-  configure(commonDebConfig(true, false))
+  configure(commonDebConfig(true, false, 'x64'))
 }
 
-Closure commonRpmConfig(boolean oss, boolean jdk) {
+Closure commonRpmConfig(boolean oss, boolean jdk, String architecture) {
   return {
-    configure(commonPackageConfig('rpm', oss, jdk))
+    configure(commonPackageConfig('rpm', oss, jdk, architecture))
 
     if (oss) {
       license 'ASL 2.0'
@@ -381,20 +389,28 @@ Closure commonRpmConfig(boolean oss, boolean jdk) {
   }
 }
 
+task buildAarch64Rpm(type: Rpm) {
+  configure(commonRpmConfig(false, true, 'aarch64'))
+}
+
 task buildRpm(type: Rpm) {
-  configure(commonRpmConfig(false, true))
+  configure(commonRpmConfig(false, true, 'x64'))
+}
+
+task buildAarch64OssRpm(type: Rpm) {
+  configure(commonRpmConfig(true, true, 'aarch64'))
 }
 
 task buildOssRpm(type: Rpm) {
-  configure(commonRpmConfig(true, true))
+  configure(commonRpmConfig(true, true, 'x64'))
 }
 
 task buildNoJdkRpm(type: Rpm) {
-  configure(commonRpmConfig(false, false))
+  configure(commonRpmConfig(false, false, 'x64'))
 }
 
 task buildOssNoJdkRpm(type: Rpm) {
-  configure(commonRpmConfig(true, false))
+  configure(commonRpmConfig(true, false, 'x64'))
 }
 
 Closure dpkgExists = { it -> new File('/bin/dpkg-deb').exists() || new File('/usr/bin/dpkg-deb').exists() || new File('/usr/local/bin/dpkg-deb').exists() }
@@ -446,6 +462,8 @@ subprojects {
         final File rpmDatabase = new File(extractionDir, 'rpm-database')
         commandLine 'rpm',
           '--badreloc',
+          '--ignorearch',
+          '--ignoreos',
           '--nodeps',
           '--noscripts',
           '--notriggers',

distribution/src/bin/elasticsearch

@@ -29,7 +29,7 @@ for option in "$@"; do
 done
 
 if [ -z "$ES_TMPDIR" ]; then
-  ES_TMPDIR=`"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory`
+  ES_TMPDIR=`"$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory`
 fi
 
 # get keystore password before setting java options to avoid
@@ -52,12 +52,13 @@ fi
 #   - second, JVM options are read from jvm.options and jvm.options.d/*.options
 #   - third, JVM options from ES_JAVA_OPTS are applied
 #   - fourth, ergonomic JVM options are applied
-ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
+ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
 
 # manual parsing to find out, if process should be detached
 if [[ $DAEMONIZE = false ]]; then
   exec \
     "$JAVA" \
+    "$XSHARE" \
     $ES_JAVA_OPTS \
     -Des.path.home="$ES_HOME" \
     -Des.path.conf="$ES_PATH_CONF" \
@@ -70,6 +71,7 @@ if [[ $DAEMONIZE = false ]]; then
 else
   exec \
     "$JAVA" \
+    "$XSHARE" \
     $ES_JAVA_OPTS \
     -Des.path.home="$ES_HOME" \
     -Des.path.conf="$ES_PATH_CONF" \

distribution/src/bin/elasticsearch-cli

@@ -22,6 +22,7 @@ ES_JAVA_OPTS="-Xms4m -Xmx64m -XX:+UseSerialGC ${ES_JAVA_OPTS}"
 
 exec \
   "$JAVA" \
+  "$XSHARE" \
   $ES_JAVA_OPTS \
   -Des.path.home="$ES_HOME" \
   -Des.path.conf="$ES_PATH_CONF" \

distribution/src/bin/elasticsearch-env

@@ -67,8 +67,14 @@ if [ ! -z "$JAVA_OPTS" ]; then
   echo "pass JVM parameters via ES_JAVA_OPTS"
 fi
 
+if [[ "$("$JAVA" -version 2>/dev/null)" =~ "Unable to map CDS archive" ]]; then
+  XSHARE="-Xshare:off"
+else
+  XSHARE="-Xshare:auto"
+fi
+
 # check the Java version
-"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.java_version_checker.JavaVersionChecker
+"$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.java_version_checker.JavaVersionChecker
 
 export HOSTNAME=$HOSTNAME
 

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java

@@ -87,6 +87,7 @@ final class JvmErgonomics {
         final List<String> command = Stream.of(
             Stream.of(java),
             userDefinedJvmOptions.stream(),
+            Stream.of("-Xshare:off"),
             Stream.of("-XX:+PrintFlagsFinal"),
             Stream.of("-version")
         ).reduce(Stream::concat).get().collect(Collectors.toUnmodifiableList());

gradle/runtime-jdk-provision.gradle

@@ -1,3 +1,4 @@
+import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.info.BuildParams
@@ -9,6 +10,7 @@ jdks {
         vendor = VersionProperties.bundledJdkVendor
         version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase())
         platform = OS.current().name().toLowerCase()
+        architecture = Architecture.current().name().toLowerCase()
     }
 }
 

settings.gradle

@@ -27,7 +27,9 @@ List projects = [
   'distribution:archives:darwin-tar',
   'distribution:archives:oss-no-jdk-darwin-tar',
   'distribution:archives:no-jdk-darwin-tar',
+  'distribution:archives:oss-linux-aarch64-tar',
   'distribution:archives:oss-linux-tar',
+  'distribution:archives:linux-aarch64-tar',
   'distribution:archives:linux-tar',
   'distribution:archives:oss-no-jdk-linux-tar',
   'distribution:archives:no-jdk-linux-tar',
@@ -36,11 +38,15 @@ List projects = [
   'distribution:docker:docker-export',
   'distribution:docker:oss-docker-build-context',
   'distribution:docker:oss-docker-export',
+  'distribution:packages:aarch64-oss-deb',
   'distribution:packages:oss-deb',
+  'distribution:packages:aarch64-deb',
   'distribution:packages:deb',
   'distribution:packages:oss-no-jdk-deb',
   'distribution:packages:no-jdk-deb',
+  'distribution:packages:aarch64-oss-rpm',
   'distribution:packages:oss-rpm',
+  'distribution:packages:aarch64-rpm',
   'distribution:packages:rpm',
   'distribution:packages:oss-no-jdk-rpm',
   'distribution:packages:no-jdk-rpm',

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -65,8 +65,15 @@ public class XPackSettings {
     public static final Setting<Boolean> GRAPH_ENABLED = Setting.boolSetting("xpack.graph.enabled", true, Setting.Property.NodeScope);
 
     /** Setting for enabling or disabling machine learning. Defaults to true. */
-    public static final Setting<Boolean> MACHINE_LEARNING_ENABLED = Setting.boolSetting("xpack.ml.enabled", true,
-            Setting.Property.NodeScope);
+    public static final Setting<Boolean> MACHINE_LEARNING_ENABLED = Setting.boolSetting(
+        "xpack.ml.enabled",
+        "aarch64".equals(System.getProperty("os.arch")) ? false : true,
+        value -> {
+            if (value && "aarch64".equals(System.getProperty("os.arch"))) {
+                throw new IllegalArgumentException("[xpack.ml.enabled] can not be set to [true] on [aarch64]");
+            }
+        },
+        Setting.Property.NodeScope);
 
     /** Setting for enabling or disabling rollup. Defaults to true. */
     public static final Setting<Boolean> ROLLUP_ENABLED = Setting.boolSetting("xpack.rollup.enabled", true,

x-pack/plugin/sql/src/main/bin/elasticsearch-sql-cli

@@ -12,5 +12,6 @@ CLI_JAR=$(ls "$ES_HOME"/bin/elasticsearch-sql-cli-*.jar)
 
 exec \
   "$JAVA" \
+  "$XSHARE" \
   -jar "$CLI_JAR" \
   "$@"