Use Cloudflare's zlib in Docker images (#81245)

Closes #81208. Elasticsearch uses zlib for two purposes:    *
Compression of stored fields with `index.codec: best_compression`,     
which we use for observability and security data.    * Request /
response compression. Historically, zlib was packaged within the JDK, so
that users wouldn't have to have zlib installed for basic usage of Java.
However, the original zlib optimizes for portability and misses a number
of important optimizations such as leveraging vectorization support for
x86 and ARM architectures. Several forks have been created in order to
address this. Since version 9, the JDK uses the system's zlib when
available and falls back to the zlib that is packaged within the JDK if
a system zlib cannot be found. This commit changes the Docker image to
install the Cloudflare fork of zlib, and run Java using the fork instead
of the original zlib, so that users of the Docker image can get better
performance. Other ES distribution types are out-of-scope, since
configuring the JVM to use an alternative zlib requires an environment
config as well as installed another zlib, and Docker is the only
distribution type where we can control both.

distribution/docker/build.gradle

@@ -16,6 +16,8 @@ apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
 apply plugin: 'elasticsearch.rest-resources'
 
+ext.cloudflareZlibVersion = '1.2.8'
+
 String buildId = providers.systemProperty('build.id').forUseAtConfigurationTime().getOrNull()
 boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false
 
@@ -34,6 +36,15 @@ repositories {
     content { includeGroup 'krallin' }
   }
 
+  ivy {
+    url 'https://github.com/'
+    patternLayout {
+      artifact '/[organisation]/[module]/archive/refs/tags/v[revision].[ext]'
+    }
+    metadataSources { artifact() }
+    content { includeGroup 'cloudflare' }
+  }
+
   // Cloud builds bundle some beats
   ivy {
     if (useLocalArtifacts) {
@@ -63,6 +74,7 @@ configurations {
   nonRepositoryPlugins
   filebeat
   metricbeat
+  cloudflareZlib
 }
 
 String beatsArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'x86_64'
@@ -77,6 +89,7 @@ dependencies {
   nonRepositoryPlugins project(path: ':plugins', configuration: 'nonRepositoryPlugins')
   filebeat "beats:filebeat:${VersionProperties.elasticsearch}:${beatsArch}@tar.gz"
   metricbeat "beats:metricbeat:${VersionProperties.elasticsearch}:${beatsArch}@tar.gz"
+  cloudflareZlib "cloudflare:zlib:${cloudflareZlibVersion}@tar.gz"
 }
 
 ext.expansions = { Architecture architecture, DockerBase base ->
@@ -100,7 +113,8 @@ ext.expansions = { Architecture architecture, DockerBase base ->
     'docker_base'        : base.name().toLowerCase(),
     'version'            : VersionProperties.elasticsearch,
     'major_minor_version': "${major}.${minor}",
-    'retry'              : ShellRetry
+    'retry'              : ShellRetry,
+    'zlib_version'       : cloudflareZlibVersion
   ]
 }
 
@@ -259,6 +273,7 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null
 
         from configurations.repositoryPlugins
+
         if (includeBeats) {
           from configurations.filebeat
           from configurations.metricbeat
@@ -289,7 +304,10 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
     from(tarTree("${project.buildDir}/distributions/${archiveName}.tar.gz")) {
       eachFile { FileCopyDetails details ->
         if (details.name.equals("Dockerfile")) {
-          filter { it.replaceAll('^RUN curl.*artifacts-no-kpi.*$', "COPY ${distributionName} /tmp/elasticsearch.tar.gz") }
+          filter { String filename ->
+            String result = filename.replaceAll('^RUN curl.*artifacts-no-kpi.*$', "COPY ${distributionName} /tmp/elasticsearch.tar.gz")
+            return result.replaceAll('^RUN curl.*cloudflare/zlib.*$', "COPY zlib-${cloudflareZlibVersion}.tar.gz /tmp")
+          }
         }
       }
     }
@@ -302,6 +320,8 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
       from configurations.dockerSource
     }
 
+    from configurations.cloudflareZlib
+
     if (base == DockerBase.IRON_BANK) {
       from (configurations.tini) {
         rename { _ -> 'tini' }

distribution/docker/src/docker/Dockerfile

@@ -20,28 +20,51 @@
 */ %>
 
 <% if (docker_base == 'iron_bank') { %>
-################################################################################
-# Build stage 0 `builder`:
-# Extract Elasticsearch artifact
-################################################################################
-
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8
 ARG BASE_TAG=8.4
+<% } %>
+
+################################################################################
+# Build stage 0 `zlib`:
+# Compile zlib for the current architecture
+################################################################################
+
+FROM ${base_image} AS zlib
+
+<% if (docker_base == 'default' || docker_base == 'cloud') { %>
+RUN <%= retry.loop(package_manager, "${package_manager} update && DEBIAN_FRONTEND=noninteractive ${package_manager} install -y curl gcc make") %>
+<% } else { %>
+RUN <%= retry.loop(package_manager, "${package_manager} install -y curl gcc make tar gzip") %>
+<% } %>
+
+<%
+// Fetch the zlib source.  Keep this command on one line - it is replaced
+// with a `COPY` during local builds.
+%>
+WORKDIR /tmp
+RUN curl --retry 10 -S -L -o zlib-${zlib_version}.tar.gz https://github.com/cloudflare/zlib/archive/refs/tags/v${zlib_version}.tar.gz
+RUN echo "7e393976368975446b263ae4143fb404bc33bf3b436e72007700b5b88e5be332cd461cdec42d31a4b6dffdca2368550f01b9fa1165d81c0aa818bbf2b1ac191e zlib-${zlib_version}.tar.gz" \\
+      | sha512sum --check -
+RUN tar xf zlib-${zlib_version}.tar.gz
+
+WORKDIR /tmp/zlib-${zlib_version}
+RUN ./configure --prefix=/usr/local/cloudflare-zlib && make && make install
+
+################################################################################
+# Build stage 1 `builder`:
+# Extract Elasticsearch artifact
+################################################################################
 
 FROM ${base_image} AS builder
 
+<% if (docker_base == 'iron_bank') { %>
 # `tini` is a tiny but valid init for containers. This is used to cleanly
 # control how ES and any child processes are shut down.
 COPY tini /bin/tini
 RUN chmod 0555 /bin/tini
 
 <% } else { %>
-################################################################################
-# Build stage 0 `builder`:
-# Extract Elasticsearch artifact
-################################################################################
-FROM ${base_image} AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 <% if (docker_base == 'default' || docker_base == 'cloud') { %>
@@ -147,9 +170,10 @@ RUN chmod -R 0555 /opt/plugins
 <% } %>
 
 ################################################################################
-# Build stage 1 (the actual Elasticsearch image):
+# Build stage 2 (the actual Elasticsearch image):
 #
-# Copy elasticsearch from stage 0
+# Copy zlib from stage 2
+# Copy elasticsearch from stage 1
 # Add entrypoint
 ################################################################################
 
@@ -207,6 +231,7 @@ WORKDIR /usr/share/elasticsearch
 COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch
 
 COPY --from=builder --chown=0:0 /bin/tini /bin/tini
+COPY --from=zlib --chown=0:0 /usr/local/cloudflare-zlib /usr/local/cloudflare-zlib
 
 <% if (docker_base == 'default' || docker_base == 'cloud') { %>
 COPY --from=builder --chown=0:0 /etc/ssl/certs/java/cacerts /etc/ssl/certs/java/cacerts
@@ -217,6 +242,7 @@ COPY --from=builder --chown=0:0 /opt /opt
 <% } %>
 
 ENV PATH /usr/share/elasticsearch/bin:\$PATH
+ENV ES_ZLIB_PATH /usr/local/cloudflare-zlib/lib
 
 COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 

distribution/src/bin/elasticsearch

@@ -52,6 +52,14 @@ if [ -z "$LIBFFI_TMPDIR" ]; then
   export LIBFFI_TMPDIR
 fi
 
+if [ -n "$ES_ZLIB_PATH" ]; then
+  if [ ! -d "$ES_ZLIB_PATH" ]; then
+    echo "zlib path specified in ES_ZLIB_PATH does not exist or is not a directory: $ES_ZLIB_PATH" >&2
+    exit 1
+  fi
+  export LD_LIBRARY_PATH="$ES_ZLIB_PATH:$LD_LIBRARY_PATH"
+fi
+
 # get keystore password before setting java options to avoid
 # conflicting GC configurations for the keystore tools
 unset KEYSTORE_PASSWORD

docs/changelog/81245.yaml

@@ -0,0 +1,6 @@
+pr: 81245
+summary: Use Cloudflare's zlib in Docker images
+area: Packaging
+type: enhancement
+issues:
+ - 81208

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -415,6 +415,17 @@ public class DockerTests extends PackagingTestCase {
         runElasticsearchTestsAsElastic(PASSWORD);
     }
 
+    /**
+     * Check that the JDK uses the Cloudflare zlib, instead of the default one.
+     */
+    public void test060JavaUsesCloudflareZlib() {
+        waitForElasticsearch(installation, "elastic", PASSWORD);
+
+        final String output = sh.run("bash -c 'pmap -p $(pidof java)'").stdout;
+
+        assertThat("Expected java to be using cloudflare-zlib", output, containsString("cloudflare-zlib"));
+    }
+
     /**
      * Check that the default config can be overridden using a bind mount, and that env vars are respected
      */