Transform docker log4j properties at build time (#71346)

For the Docker distribution, we transform the archive distribution's
log4j2 config so that all messages are logged to the console by default.
However this transformation step happens when the Docker image is built,
which means that the source for the transformation must be included in
the Docker context.

Improve this by making the archive distribution's log4j config available
as an artifact in the build, then simply copying it into the Docker context.
The transform logic has been reimplemented as a simple copy filter.
Consequently, the `transform-log4j-config` project has been removed.

buildSrc/build.gradle

@@ -111,6 +111,7 @@ dependencies {
   testFixturesApi gradleTestKit()
   testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
   testImplementation 'org.mockito:mockito-core:1.9.5'
+  testImplementation "org.hamcrest:hamcrest:${props.getProperty('hamcrest')}"
   integTestImplementation('org.spockframework:spock-core:1.3-groovy-2.5') {
     exclude module: "groovy"
   }

distribution/docker/transform-log4j-config/src/main/java/org/elasticsearch/transform/log4j/TransformLog4jConfig.java → buildSrc/src/main/java/org/elasticsearch/gradle/docker/TransformLog4jConfigFilter.java

@@ -6,49 +6,26 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.transform.log4j;
+package org.elasticsearch.gradle.docker;
 
+import org.apache.commons.io.IOUtils;
+
+import java.io.FilterReader;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
+import java.io.Reader;
+import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.List;
 
-/**
- * This class takes in a log4j configuration file, and transform it into a config that
- * writes everything to the console. This is useful when running Elasticsearch in a Docker
- * container, where the Docker convention is to log to stdout / stderr and let the
- * orchestration layer direct the output.
- */
-public class TransformLog4jConfig {
-
-    public static void main(String[] args) throws IOException {
-        List<String> lines = getConfigFile(args);
-
-        final List<String> output = skipBlanks(transformConfig(lines));
-
-        output.forEach(System.out::println);
+public class TransformLog4jConfigFilter extends FilterReader {
+    public TransformLog4jConfigFilter(Reader in) throws IOException {
+        super(new StringReader(transform(in)));
     }
 
-    private static List<String> getConfigFile(String[] args) throws IOException {
-        if (args.length != 1) {
-            System.err.println("ERROR: Must supply a single argument, the file to process");
-            System.exit(1);
-        }
-
-        Path configPath = Path.of(args[0]);
-
-        if (Files.exists(configPath) == false) {
-            System.err.println("ERROR: [" + configPath + "] does not exist");
-            System.exit(1);
-        }
-
-        if (Files.isReadable(configPath) == false) {
-            System.err.println("ERROR: [" + configPath + "] exists but is not readable");
-            System.exit(1);
-        }
-
-        return Files.readAllLines(configPath);
+    private static String transform(Reader reader) throws IOException {
+        final List<String> inputLines = IOUtils.readLines(reader);
+        final List<String> outputLines = skipBlanks(transformConfig(inputLines));
+        return String.join("\n", outputLines);
     }
 
     /** Squeeze multiple empty lines into a single line. */

buildSrc/src/main/resources/checkstyle_ide_fragment.xml

@@ -15,7 +15,8 @@
         <property name="severity" value="warning" />
         <!-- Exclude short methods from this check - we don't want to have to document getters -->
         <property name="minLineCount" value="2" />
-        <property name="allowedAnnotations" value="Override,Before,BeforeClass,After,AfterClass,Inject" />
+        <property name="allowedAnnotations" value="Override,Before,BeforeClass,After,AfterClass,Inject,TaskAction" />
+        <property name="ignoreMethodNamesRegex" value="^main$"/>
         <message key="javadoc.missing" value="Public methods should be documented" />
     </module>
 

distribution/docker/transform-log4j-config/src/test/java/org/elasticsearch/transform/log4j/TransformLog4jConfigTests.java → buildSrc/src/test/java/org/elasticsearch/gradle/docker/TransformLog4jConfigFilterTests.java

@@ -6,16 +6,15 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.transform.log4j;
+package org.elasticsearch.gradle.docker;
 
-import junit.framework.TestCase;
+import org.elasticsearch.gradle.test.GradleUnitTestCase;
 
 import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertThat;
 
-public class TransformLog4jConfigTests extends TestCase {
+public class TransformLog4jConfigFilterTests extends GradleUnitTestCase {
 
     /**
      * Check that the transformer doesn't explode when given an empty file.
@@ -138,12 +137,12 @@ public class TransformLog4jConfigTests extends TestCase {
 
         List<String> expected = List.of("status = error", "", "rootLogger.level = info");
 
-        final List<String> transformed = TransformLog4jConfig.skipBlanks(input);
+        final List<String> transformed = TransformLog4jConfigFilter.skipBlanks(input);
         assertThat(transformed, equalTo(expected));
     }
 
     private void runTest(List<String> input, List<String> expected) {
-        final List<String> transformed = TransformLog4jConfig.transformConfig(input);
+        final List<String> transformed = TransformLog4jConfigFilter.transformConfig(input);
 
         assertThat(transformed, equalTo(expected));
     }

distribution/build.gradle

@@ -21,6 +21,11 @@ import java.nio.file.Path
 plugins {
   id 'base'
 }
+
+configurations {
+  log4jConfig
+}
+
 /*****************************************************************************
  *                  Third party dependencies report                          *
  *****************************************************************************/
@@ -633,3 +638,12 @@ subprojects {
     "${configuration.name}" project(path: subproject.path, configuration:'default')
   }
 }
+
+// This artifact makes it possible for other projects to pull
+// in the final log4j2.properties configuration, as it appears in the
+// archive distribution.
+artifacts.add('log4jConfig', file("${defaultOutputs}/log4j2.properties")) {
+  type 'file'
+  name 'log4j2.properties'
+  builtBy 'buildDefaultLog4jConfig'
+}

distribution/docker/build.gradle

@@ -4,6 +4,7 @@ import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.docker.DockerBuildTask
 import org.elasticsearch.gradle.docker.ShellRetry
+import org.elasticsearch.gradle.docker.TransformLog4jConfigFilter
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
 
@@ -36,14 +37,14 @@ testFixtures.useFixture()
 configurations {
   aarch64DockerSource
   dockerSource
-  transformLog4jJar
+  log4jConfig
   tini
 }
 
 dependencies {
   aarch64DockerSource project(path: ":distribution:archives:linux-aarch64-tar", configuration: 'default')
   dockerSource project(path: ":distribution:archives:linux-tar", configuration: 'default')
-  transformLog4jJar project(path: ":distribution:docker:transform-log4j-config", configuration: 'default')
+  log4jConfig project(path: ":distribution", configuration: 'log4jConfig')
   tini 'krallin:tini:0.19.0@tini-amd64'
 }
 
@@ -115,7 +116,7 @@ ext.dockerBuildContext = { Architecture architecture, DockerBase base ->
       }
     }
 
-    from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) {
+    from(projectDir.resolve("src/docker/Dockerfile")) {
       expand(varExpansions)
       filter SquashNewlinesFilter
     }
@@ -197,23 +198,24 @@ tasks.named("composePull").configure {
 }
 
 void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
-  String binDirectory = base == DockerBase.IRON_BANK ? 'scripts' : 'bin'
+  String configDirectory = base == DockerBase.IRON_BANK ? 'scripts' : 'config'
   String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
 
-  final TaskProvider<DockerBuildTask> buildContextTask =
-    tasks.register(taskName('build', architecture, base, 'DockerContext'), Tar) {
-      archiveExtension = 'tar.gz'
-      compression = Compression.GZIP
-      archiveClassifier = "docker-build-context${arch}"
-      archiveBaseName = "elasticsearch${base.suffix}"
-      with dockerBuildContext(architecture, base)
+  tasks.register(taskName('build', architecture, base, 'DockerContext'), Tar) {
+    archiveExtension = 'tar.gz'
+    compression = Compression.GZIP
+    archiveClassifier = "docker-build-context${arch}"
+    archiveBaseName = "elasticsearch${base.suffix}"
+    with dockerBuildContext(architecture, base)
 
-      into (binDirectory) {
-        from configurations.transformLog4jJar
+    into(configDirectory) {
+      from(configurations.log4jConfig) {
+        filter TransformLog4jConfigFilter
       }
-
-      onlyIf { Architecture.current() == architecture }
     }
+
+    onlyIf { Architecture.current() == architecture }
+  }
 }
 
 void addUnpackDockerContextTask(Architecture architecture, DockerBase base) {

distribution/docker/src/docker/Dockerfile

@@ -228,13 +228,13 @@ RUN tar -zxf /opt/elasticsearch.tar.gz --strip-components=1
 
 # The distribution includes a `config` directory, no need to create it
 COPY ${config_dir}/elasticsearch.yml config/
-COPY ${bin_dir}/transform-log4j-config-${version}.jar /tmp/
+COPY ${config_dir}/log4j2.properties config/log4j2.docker.properties
 
 # 1. Configure the distribution for Docker
 # 2. Ensure directories are created. Most already are, but make sure
 # 3. Apply correct permissions
 # 4. Move the distribution's default logging config aside
-# 5. Generate a docker logging config, to be used by default
+# 5. Move the generated docker logging config so that it is the default
 # 6. Apply more correct permissions
 # 7. The JDK's directories' permissions don't allow `java` to be executed under a different
 #    group to the default. Fix this.
@@ -245,7 +245,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
     mkdir -p config/jvm.options.d data logs plugins && \\
     chmod 0775 config config/jvm.options.d data logs plugins && \\
     mv config/log4j2.properties config/log4j2.file.properties && \\
-    jdk/bin/java -jar /tmp/transform-log4j-config-${version}.jar config/log4j2.file.properties > config/log4j2.properties && \\
+    mv config/log4j2.docker.properties config/log4j2.properties && \\
     chmod 0660 config/elasticsearch.yml config/log4j2*.properties && \\
     find ./jdk -type d -exec chmod 0755 {} + && \\
     find . -xdev -perm -4000 -exec chmod ug-s {} + && \\

distribution/docker/transform-log4j-config/build.gradle

@@ -1,29 +0,0 @@
-apply plugin: 'elasticsearch.build'
-
-repositories {
-  mavenCentral()
-}
-
-dependencies {
-  testImplementation "junit:junit:${versions.junit}"
-  testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
-}
-
-tasks.named('jar').configure {
-  manifest {
-    attributes 'Main-Class': 'org.elasticsearch.transform.log4j.TransformLog4jConfig'
-  }
-}
-
-// This tests depend on ES core
-disableTasks('forbiddenApisMain', 'forbiddenApisTest')
-
-tasks.named('testingConventions').configure {
-  naming.clear()
-  naming {
-    Tests {
-      baseClass 'junit.framework.TestCase'
-    }
-  }
-}
-

settings.gradle

@@ -32,7 +32,6 @@ List projects = [
   'distribution:docker:docker-export',
   'distribution:docker:ironbank-docker-aarch64-export',
   'distribution:docker:ironbank-docker-export',
-  'distribution:docker:transform-log4j-config',
   'distribution:docker:ubi-docker-aarch64-export',
   'distribution:docker:ubi-docker-export',
   'distribution:packages:aarch64-deb',