Introduce Docker images build (#36246)

This commit introduces the building of the Docker images as bonafide
packaging formats alongside our existing archive and packaging
distributions. This build is migrated from a dedicated repository, and
converted to Gradle in the process.

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -51,6 +51,7 @@ import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.api.tasks.javadoc.Javadoc
 import org.gradle.internal.jvm.Jvm
 import org.gradle.process.ExecResult
+import org.gradle.process.ExecSpec
 import org.gradle.util.GradleVersion
 
 import java.nio.charset.StandardCharsets
@@ -232,6 +233,95 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.java9Home = project.rootProject.ext.java9Home
     }
 
+    static void requireDocker(final Task task) {
+        final Project rootProject = task.project.rootProject
+        if (rootProject.hasProperty('requiresDocker') == false) {
+            /*
+             * This is our first time encountering a task that requires Docker. We will add an extension that will let us track the tasks
+             * that register as requiring Docker. We will add a delayed execution that when the task graph is ready if any such tasks are
+             * in the task graph, then we check two things:
+             *  - the Docker binary is available
+             *  - we can execute a Docker command that requires privileges
+             *
+             *  If either of these fail, we fail the build.
+             */
+            final boolean buildDocker
+            final String buildDockerProperty = System.getProperty("build.docker")
+            if (buildDockerProperty == null || buildDockerProperty == "true") {
+                buildDocker = true
+            } else if (buildDockerProperty == "false") {
+                buildDocker = false
+            } else {
+                throw new IllegalArgumentException(
+                        "expected build.docker to be unset or one of \"true\" or \"false\" but was [" + buildDockerProperty + "]")
+            }
+            rootProject.rootProject.ext.buildDocker = buildDocker
+            rootProject.rootProject.ext.requiresDocker = []
+            rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
+                // check if the Docker binary exists and record its path
+                final List<String> maybeDockerBinaries = ['/usr/bin/docker2', '/usr/local/bin/docker2']
+                final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() }
+
+                int exitCode
+                String dockerErrorOutput
+                if (dockerBinary == null) {
+                    exitCode = -1
+                    dockerErrorOutput = null
+                } else {
+                    // the Docker binary executes, check that we can execute a privileged command
+                    final ByteArrayOutputStream output = new ByteArrayOutputStream()
+                    final ExecResult result = LoggedExec.exec(rootProject, { ExecSpec it ->
+                        it.commandLine dockerBinary, "images"
+                        it.errorOutput = output
+                        it.ignoreExitValue = true
+                    })
+                    if (result.exitValue == 0) {
+                        return
+                    }
+                    exitCode = result.exitValue
+                    dockerErrorOutput = output.toString()
+                }
+                final List<String> tasks =
+                        ((List<Task>)rootProject.requiresDocker).findAll { taskGraph.hasTask(it) }.collect { "  ${it.path}".toString()}
+                if (tasks.isEmpty() == false) {
+                    /*
+                     * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not
+                     * exist or because execution of a privileged Docker command failed.
+                     */
+                    String message
+                    if (dockerBinary == null) {
+                        message = String.format(
+                                Locale.ROOT,
+                                "Docker (checked [%s]) is required to run the following task%s: \n%s",
+                                maybeDockerBinaries.join(","),
+                                tasks.size() > 1 ? "s" : "",
+                                tasks.join('\n'))
+                    } else {
+                        assert exitCode > 0 && dockerErrorOutput != null
+                        message = String.format(
+                                Locale.ROOT,
+                                "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" +
+                                        "the problem is that Docker exited with exit code [%d] with standard error output [%s]",
+                                dockerBinary,
+                                tasks.size() > 1 ? "s" : "",
+                                tasks.join('\n'),
+                                exitCode,
+                                dockerErrorOutput.trim())
+                    }
+                    throw new GradleException(
+                            message + "\nyou can address this by attending to the reported issue, "
+                                    + "removing the offending tasks from being executed, "
+                                    + "or by passing -Dbuild.docker=false")
+                }
+            }
+        }
+        if (rootProject.buildDocker) {
+            rootProject.requiresDocker.add(task)
+        } else {
+            task.enabled = false
+        }
+    }
+
     private static String findCompilerJavaHome() {
         String compilerJavaHome = System.getenv('JAVA_HOME')
         final String compilerJavaProperty = System.getProperty('compiler.java')

distribution/docker/build.gradle

@@ -0,0 +1,106 @@
+import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.LoggedExec
+import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.VersionProperties
+
+apply plugin: 'base'
+
+configurations {
+  dockerPlugins
+  dockerSource
+  ossDockerSource
+}
+
+dependencies {
+  dockerPlugins project(path: ":plugins:ingest-geoip", configuration: 'zip')
+  dockerPlugins project(path: ":plugins:ingest-user-agent", configuration: 'zip')
+  dockerSource project(path: ":distribution:archives:tar")
+  ossDockerSource project(path: ":distribution:archives:oss-tar")
+}
+
+ext.expansions = { oss ->
+  return [
+    'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz",
+    'jdkUrl' : 'https://download.java.net/java/GA/jdk11/13/GPL/openjdk-11.0.1_linux-x64_bin.tar.gz',
+    'jdkVersion' : '11.0.1',
+    'license': oss ? 'Apache-2.0' : 'Elastic License',
+    'ingest-geoip' : "ingest-geoip-${VersionProperties.elasticsearch}.zip",
+    'ingest-user-agent' : "ingest-user-agent-${VersionProperties.elasticsearch}.zip",
+    'version' : VersionProperties.elasticsearch
+  ]
+}
+
+private static String files(final boolean oss) {
+  return "build/${ oss ? 'oss-' : ''}docker"
+}
+
+private static String taskName(final String prefix, final boolean oss, final String suffix) {
+  return "${prefix}${oss ? 'Oss' : ''}${suffix}"
+}
+
+void addCopyDockerContextTask(final boolean oss) {
+  task(taskName("copy", oss, "DockerContext"), type: Sync) {
+    into files(oss)
+
+    into('bin') {
+      from 'src/docker/bin'
+    }
+
+    into('config') {
+      from 'src/docker/config'
+    }
+
+    if (oss) {
+      from configurations.ossDockerSource
+    } else {
+      from configurations.dockerSource
+    }
+
+    from configurations.dockerPlugins
+  }
+}
+
+void addCopyDockerfileTask(final boolean oss) {
+  task(taskName("copy", oss, "Dockerfile"), type: Copy) {
+    mustRunAfter(taskName("copy", oss, "DockerContext"))
+    into files(oss)
+
+    from('src/docker/Dockerfile') {
+      MavenFilteringHack.filter(it, expansions(oss))
+    }
+  }
+}
+
+void addBuildDockerImage(final boolean oss) {
+  final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) {
+    dependsOn taskName("copy", oss, "DockerContext")
+    dependsOn taskName("copy", oss, "Dockerfile")
+    List<String> tags
+    if (oss) {
+      tags = [ "docker.elastic.co/elasticsearch/elasticsearch-oss:${VersionProperties.elasticsearch}" ]
+    } else {
+      tags = [
+        "elasticsearch:${VersionProperties.elasticsearch}",
+        "docker.elastic.co/elasticsearch/elasticsearch:${VersionProperties.elasticsearch}",
+        "docker.elastic.co/elasticsearch/elasticsearch-full:${VersionProperties.elasticsearch}"
+      ]
+    }
+    executable 'docker'
+    final List<String> dockerArgs = ['build', files(oss), '--pull']
+    for (final String tag : tags) {
+      dockerArgs.add('--tag')
+      dockerArgs.add(tag)
+    }
+    args dockerArgs.toArray()
+  }
+  BuildPlugin.requireDocker(buildDockerImageTask)
+}
+
+for (final boolean oss : [false, true]) {
+  addCopyDockerContextTask(oss)
+  addCopyDockerfileTask(oss)
+  addBuildDockerImage(oss)
+}
+
+assemble.dependsOn "buildOssDockerImage"
+assemble.dependsOn "buildDockerImage"

distribution/docker/src/docker/Dockerfile

@@ -0,0 +1,92 @@
+################################################################################
+# This Dockerfile was generated from the template at distribution/src/docker/Dockerfile
+#
+# Beginning of multi stage Dockerfile
+################################################################################
+
+################################################################################
+# Build stage 0 `builder`:
+# Extract elasticsearch artifact
+# Install required plugins
+# Set gid=0 and make group perms==owner perms
+################################################################################
+
+FROM centos:7 AS builder
+
+ENV PATH /usr/share/elasticsearch/bin:$PATH
+ENV JAVA_HOME /opt/jdk-${jdkVersion}
+
+RUN curl -s ${jdkUrl} | tar -C /opt -zxf -
+
+# Replace OpenJDK's built-in CA certificate keystore with the one from the OS
+# vendor. The latter is superior in several ways.
+# REF: https://github.com/elastic/elasticsearch-docker/issues/171
+RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /opt/jdk-${jdkVersion}/lib/security/cacerts
+
+RUN yum install -y unzip which
+
+RUN groupadd -g 1000 elasticsearch && \
+    adduser -u 1000 -g 1000 -d /usr/share/elasticsearch elasticsearch
+
+WORKDIR /usr/share/elasticsearch
+
+COPY ${elasticsearch} ${ingest-geoip} ${ingest-user-agent} /opt/
+RUN tar zxf /opt/${elasticsearch} --strip-components=1
+RUN elasticsearch-plugin install --batch file:///opt/${ingest-geoip}
+RUN elasticsearch-plugin install --batch file:///opt/${ingest-user-agent}
+RUN mkdir -p config data logs
+RUN chmod 0775 config data logs
+COPY config/elasticsearch.yml config/log4j2.properties config/
+
+
+################################################################################
+# Build stage 1 (the actual elasticsearch image):
+# Copy elasticsearch from stage 0
+# Add entrypoint
+################################################################################
+
+FROM centos:7
+
+ENV ELASTIC_CONTAINER true
+ENV JAVA_HOME /opt/jdk-${jdkVersion}
+
+COPY --from=builder /opt/jdk-${jdkVersion} /opt/jdk-${jdkVersion}
+
+RUN yum update -y && \
+    yum install -y nc unzip wget which && \
+    yum clean all
+
+RUN groupadd -g 1000 elasticsearch && \
+    adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \
+    chmod 0775 /usr/share/elasticsearch && \
+    chgrp 0 /usr/share/elasticsearch
+
+WORKDIR /usr/share/elasticsearch
+COPY --from=builder --chown=1000:0 /usr/share/elasticsearch /usr/share/elasticsearch
+ENV PATH /usr/share/elasticsearch/bin:$PATH
+
+COPY --chown=1000:0 bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+
+# Openshift overrides USER and uses ones with randomly uid>1024 and gid=0
+# Allow ENTRYPOINT (and ES) to run even with a different user
+RUN chgrp 0 /usr/local/bin/docker-entrypoint.sh && \
+    chmod g=u /etc/passwd && \
+    chmod 0775 /usr/local/bin/docker-entrypoint.sh
+
+EXPOSE 9200 9300
+
+LABEL org.label-schema.schema-version="1.0" \
+  org.label-schema.vendor="Elastic" \
+  org.label-schema.name="elasticsearch" \
+  org.label-schema.version="${version}" \
+  org.label-schema.url="https://www.elastic.co/products/elasticsearch" \
+  org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \
+  license="${license}"
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+# Dummy overridable parameter parsed by entrypoint
+CMD ["eswrapper"]
+
+################################################################################
+# End of multi-stage Dockerfile
+################################################################################

distribution/docker/src/docker/bin/docker-entrypoint.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+set -e
+
+# Files created by Elasticsearch should always be group writable too
+umask 0002
+
+run_as_other_user_if_needed() {
+  if [[ "$(id -u)" == "0" ]]; then
+    # If running as root, drop to specified UID and run command
+    exec chroot --userspec=1000 / "${@}"
+  else
+    # Either we are running in Openshift with random uid and are a member of the root group
+    # or with a custom --user
+    exec "${@}"
+  fi
+}
+
+# Allow user specify custom CMD, maybe bin/elasticsearch itself
+# for example to directly specify `-E` style parameters for elasticsearch on k8s
+# or simply to run /bin/bash to check the image
+if [[ "$1" != "eswrapper" ]]; then
+  if [[ "$(id -u)" == "0" && $(basename "$1") == "elasticsearch" ]]; then
+    # centos:7 chroot doesn't have the `--skip-chdir` option and
+    # changes our CWD.
+    # Rewrite CMD args to replace $1 with `elasticsearch` explicitly,
+    # so that we are backwards compatible with the docs
+    # from the previous Elasticsearch versions<6
+    # and configuration option D:
+    # https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docker.html#_d_override_the_image_8217_s_default_ulink_url_https_docs_docker_com_engine_reference_run_cmd_default_command_or_options_cmd_ulink
+    # Without this, user could specify `elasticsearch -E x.y=z` but
+    # `bin/elasticsearch -E x.y=z` would not work.
+    set -- "elasticsearch" "${@:2}"
+    # Use chroot to switch to UID 1000
+    exec chroot --userspec=1000 / "$@"
+  else
+    # User probably wants to run something else, like /bin/bash, with another uid forced (Openshift?)
+    exec "$@"
+  fi
+fi
+
+# Parse Docker env vars to customize Elasticsearch
+#
+# e.g. Setting the env var cluster.name=testcluster
+#
+# will cause Elasticsearch to be invoked with -Ecluster.name=testcluster
+#
+# see https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html#_setting_default_settings
+
+declare -a es_opts
+
+while IFS='=' read -r envvar_key envvar_value
+do
+  # Elasticsearch settings need to have at least two dot separated lowercase
+  # words, e.g. `cluster.name`, except for `processors` which we handle
+  # specially
+  if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ || "$envvar_key" == "processors" ]]; then
+    if [[ ! -z $envvar_value ]]; then
+      es_opt="-E${envvar_key}=${envvar_value}"
+      es_opts+=("${es_opt}")
+    fi
+  fi
+done < <(env)
+
+# The virtual file /proc/self/cgroup should list the current cgroup
+# membership. For each hierarchy, you can follow the cgroup path from
+# this file to the cgroup filesystem (usually /sys/fs/cgroup/) and
+# introspect the statistics for the cgroup for the given
+# hierarchy. Alas, Docker breaks this by mounting the container
+# statistics at the root while leaving the cgroup paths as the actual
+# paths. Therefore, Elasticsearch provides a mechanism to override
+# reading the cgroup path from /proc/self/cgroup and instead uses the
+# cgroup path defined the JVM system property
+# es.cgroups.hierarchy.override. Therefore, we set this value here so
+# that cgroup statistics are available for the container this process
+# will run in.
+export ES_JAVA_OPTS="-Des.cgroups.hierarchy.override=/ $ES_JAVA_OPTS"
+
+if [[ -d bin/x-pack ]]; then
+  # Check for the ELASTIC_PASSWORD environment variable to set the
+  # bootstrap password for Security.
+  #
+  # This is only required for the first node in a cluster with Security
+  # enabled, but we have no way of knowing which node we are yet. We'll just
+  # honor the variable if it's present.
+  if [[ -n "$ELASTIC_PASSWORD" ]]; then
+    [[ -f /usr/share/elasticsearch/config/elasticsearch.keystore ]] || (run_as_other_user_if_needed elasticsearch-keystore create)
+    if ! (run_as_other_user_if_needed elasticsearch-keystore list | grep -q '^bootstrap.password$'); then
+      (run_as_other_user_if_needed echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x 'bootstrap.password')
+    fi
+  fi
+fi
+
+if [[ "$(id -u)" == "0" ]]; then
+  # If requested and running as root, mutate the ownership of bind-mounts
+  if [[ -n "$TAKE_FILE_OWNERSHIP" ]]; then
+    chown -R 1000:0 /usr/share/elasticsearch/{data,logs}
+  fi
+fi
+
+run_as_other_user_if_needed /usr/share/elasticsearch/bin/elasticsearch "${es_opts[@]}"

distribution/docker/src/docker/config/elasticsearch.yml

@@ -0,0 +1,2 @@
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0

distribution/docker/src/docker/config/log4j2.properties

@@ -0,0 +1,9 @@
+status = error
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console

settings.gradle

@@ -20,6 +20,7 @@ List projects = [
   'distribution:archives:zip',
   'distribution:archives:oss-tar',
   'distribution:archives:tar',
+  'distribution:docker',
   'distribution:packages:oss-deb',
   'distribution:packages:deb',
   'distribution:packages:oss-rpm',