Adjustments for FIPS 140 testing (#49319)

- Don't install ingest-attachment and don't run the related docs
tests, since ingest-attachment is not supported in FIPS 140 JVMs
- Move copying extra jars and extra config files earlier on in the
node configuration so that elasticsearch-keystore and
elasticsearch-plugin that run before the node starts have all files
(policy, properties, jars) available.
- BCJSSE needs a certificate to be explicitly added in a keystore 
as a trustedcerty entry, it's not enough for it to be in privatekeyentry 
for it to be trusted
- Set the value for BuildParams.inFipsJvm configuration time

buildSrc/build.gradle

@@ -168,6 +168,9 @@ if (project != rootProject) {
   forbiddenApisTest.enabled = false
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
+  if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))){
+    test.enabled = false
+  }
 
   configurations {
     distribution

buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java

@@ -41,14 +41,12 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask {
     private final RegularFileProperty outputFile;
     private final RegularFileProperty compilerVersionFile;
     private final RegularFileProperty runtimeVersionFile;
-    private final RegularFileProperty fipsJvmFile;
 
     @Inject
     public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) {
         this.outputFile = objectFactory.fileProperty();
         this.compilerVersionFile = objectFactory.fileProperty();
         this.runtimeVersionFile = objectFactory.fileProperty();
-        this.fipsJvmFile = objectFactory.fileProperty();
     }
 
     @Input
@@ -113,11 +111,6 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask {
         return runtimeVersionFile;
     }
 
-    @OutputFile
-    public RegularFileProperty getFipsJvmFile() {
-        return fipsJvmFile;
-    }
-
     @TaskAction
     public void generate() {
         String javaVendorVersion = System.getProperty("java.vendor.version", System.getProperty("java.vendor"));
@@ -130,7 +123,6 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask {
         String runtimeJavaVersionDetails = gradleJavaVersionDetails;
         JavaVersion runtimeJavaVersionEnum = JavaVersion.current();
         File gradleJavaHome = Jvm.current().getJavaHome();
-        boolean inFipsJvm = false;
 
         try {
             if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) {
@@ -146,8 +138,6 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask {
                 if (runtimeJavaHome.exists()) {
                     runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome);
                     runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome));
-
-                    inFipsJvm = Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
                 } else {
                     throw new RuntimeException("Runtime Java home path of '" + compilerJavaHome + "' does not exist");
                 }
@@ -213,7 +203,6 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask {
 
         writeToFile(compilerVersionFile.getAsFile().get(), compilerJavaVersionEnum.name());
         writeToFile(runtimeVersionFile.getAsFile().get(), runtimeJavaVersionEnum.name());
-        writeToFile(fipsJvmFile.getAsFile().get(), Boolean.toString(inFipsJvm));
     }
 
     private void writeToFile(File file, String content) {

buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -76,14 +76,12 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
                 task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info"));
                 task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version"));
                 task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version"));
-                task.getFipsJvmFile().set(new File(project.getBuildDir(), "in-fips-jvm"));
             });
 
         PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> {
             task.getBuildInfoFile().set(generateTask.getOutputFile());
             task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile());
             task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile());
-            task.getFipsJvmFile().set(generateTask.getFipsJvmFile());
             task.setGlobalInfoListeners(extension.listeners);
         });
 
@@ -103,6 +101,7 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
             params.setIsCi(System.getenv("JENKINS_URL") != null);
             params.setIsInternal(GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null);
             params.setDefaultParallel(findDefaultParallel(project));
+            params.setInFipsJvm(isInFipsJvm());
         });
 
         project.allprojects(p -> {
@@ -153,6 +152,10 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
         return "JAVA" + version + "_HOME";
     }
 
+    private static boolean isInFipsJvm() {
+        return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
+    }
+
     private static String getResourceContents(String resourcePath) {
         try (BufferedReader reader = new BufferedReader(
             new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath))

buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java

@@ -16,7 +16,6 @@ public class PrintGlobalBuildInfoTask extends DefaultTask {
     private final RegularFileProperty buildInfoFile;
     private final RegularFileProperty compilerVersionFile;
     private final RegularFileProperty runtimeVersionFile;
-    private final RegularFileProperty fipsJvmFile;
     private List<Runnable> globalInfoListeners = new ArrayList<>();
 
     @Inject
@@ -24,7 +23,6 @@ public class PrintGlobalBuildInfoTask extends DefaultTask {
         this.buildInfoFile = objectFactory.fileProperty();
         this.compilerVersionFile = objectFactory.fileProperty();
         this.runtimeVersionFile = objectFactory.fileProperty();
-        this.fipsJvmFile = objectFactory.fileProperty();
     }
 
     @InputFile
@@ -42,11 +40,6 @@ public class PrintGlobalBuildInfoTask extends DefaultTask {
         return runtimeVersionFile;
     }
 
-    @InputFile
-    public RegularFileProperty getFipsJvmFile() {
-        return fipsJvmFile;
-    }
-
     public void setGlobalInfoListeners(List<Runnable> globalInfoListeners) {
         this.globalInfoListeners = globalInfoListeners;
     }
@@ -57,6 +50,7 @@ public class PrintGlobalBuildInfoTask extends DefaultTask {
         getLogger().quiet("Elasticsearch Build Hamster says Hello!");
         getLogger().quiet(getFileText(getBuildInfoFile()).asString());
         getLogger().quiet("  Random Testing Seed   : " + BuildParams.getTestSeed());
+        getLogger().quiet("  In FIPS 140 mode      : " + BuildParams.isInFipsJvm());
         getLogger().quiet("=======================================");
 
         setGlobalProperties();
@@ -76,7 +70,6 @@ public class PrintGlobalBuildInfoTask extends DefaultTask {
         BuildParams.init(params -> {
             params.setCompilerJavaVersion(JavaVersion.valueOf(getFileText(getCompilerVersionFile()).asString()));
             params.setRuntimeJavaVersion(JavaVersion.valueOf(getFileText(getRuntimeVersionFile()).asString()));
-            params.setInFipsJvm(Boolean.parseBoolean(getFileText(getFipsJvmFile()).asString()));
         });
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -415,6 +415,11 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         } catch (IOException e) {
             throw new UncheckedIOException("Failed to create working directory for " + this, e);
         }
+
+        copyExtraJars();
+
+        copyExtraConfigFiles();
+
         createConfiguration();
 
         if (plugins.isEmpty() == false) {
@@ -438,7 +443,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
             runElaticsearchBinScript("elasticsearch-keystore", "create");
 
             keystoreSettings.forEach((key, value) ->
-                runElaticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
+                runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
             );
 
             for (Map.Entry<String, File> entry : keystoreFiles.entrySet()) {
@@ -453,10 +458,6 @@ public class ElasticsearchNode implements TestClusterConfiguration {
 
         installModules();
 
-        copyExtraConfigFiles();
-
-        copyExtraJars();
-
         if (isSettingTrue("xpack.security.enabled")) {
             if (credentials.isEmpty()) {
                 user(Collections.emptyMap());
@@ -622,7 +623,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         credentials.add(cred);
     }
 
-    private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) {
+    private void runElasticsearchBinScriptWithInput(String input, String tool, String... args) {
         if (
             Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false &&
                 Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false
@@ -663,7 +664,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     }
 
     private void runElaticsearchBinScript(String tool, String... args) {
-        runElaticsearchBinScriptWithInput("", tool, args);
+        runElasticsearchBinScriptWithInput("", tool, args);
     }
 
     private Map<String, String> getESEnvironment() {
@@ -676,6 +677,10 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         if (systemProperties.isEmpty() == false) {
             systemPropertiesString = " " + systemProperties.entrySet().stream()
                 .map(entry -> "-D" + entry.getKey() + "=" + entry.getValue())
+                // ES_PATH_CONF is also set as an environment variable and for a reference to ${ES_PATH_CONF}
+                // to work ES_JAVA_OPTS, we need to make sure that ES_PATH_CONF before ES_JAVA_OPTS. Instead,
+                // we replace the reference with the actual value in other environment variables
+                .map(p -> p.replace("${ES_PATH_CONF}", configFile.getParent().toString()))
                 .collect(Collectors.joining(" "));
         }
         String jvmArgsString = "";

client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java

@@ -81,6 +81,7 @@ public class RestClientBuilderIntegTests extends RestClientTestCase {
     }
 
     public void testBuilderUsesDefaultSSLContext() throws Exception {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         final SSLContext defaultSSLContext = SSLContext.getDefault();
         try {
             try (RestClient client = buildRestClient()) {
@@ -109,7 +110,8 @@ public class RestClientBuilderIntegTests extends RestClientTestCase {
 
     private static SSLContext getSslContext() throws Exception {
         SSLContext sslContext = SSLContext.getInstance(getProtocol());
-        try (InputStream certFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test.crt")) {
+        try (InputStream certFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test.crt");
+             InputStream keyStoreFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test_truststore.jks")) {
             // Build a keystore of default type programmatically since we can't use JKS keystores to
             // init a KeyManagerFactory in FIPS 140 JVMs.
             KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
@@ -122,8 +124,10 @@ public class RestClientBuilderIntegTests extends RestClientTestCase {
                 new Certificate[]{certFactory.generateCertificate(certFile)});
             KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
             kmf.init(keyStore, "password".toCharArray());
+            KeyStore trustStore = KeyStore.getInstance("JKS");
+            trustStore.load(keyStoreFile, "password".toCharArray());
             TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-            tmf.init(keyStore);
+            tmf.init(trustStore);
             sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
         }
         return sslContext;

client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java

@@ -108,4 +108,8 @@ public abstract class RestClientTestCase extends RandomizedTest {
         }
         values.add(value);
     }
+
+    public static boolean inFipsJvm() {
+        return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
+    }
 }

docs/build.gradle

@@ -71,6 +71,10 @@ project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each {
   if (subproj.path.startsWith(':plugins:repository-')) {
     return
   }
+  // Do not install ingest-attachment in a FIPS 140 JVM as this is not supported
+  if (subproj.path.startsWith(':plugins:ingest-attachment') && Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))) {
+    return
+  }
   // FIXME
   subproj.afterEvaluate { // need to wait until the project has been configured
     testClusters.integTest {
@@ -89,6 +93,12 @@ buildRestTests.docs = fileTree(projectDir) {
   exclude 'README.asciidoc'
   // Broken code snippet tests
   exclude 'reference/graph/explore.asciidoc'
+  if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))) {
+    // We don't install/support this plugin in FIPS 140
+    exclude 'plugins/ingest-attachment.asciidoc'
+    // We can't conditionally control output, this would be missing the ingest-attachment plugin
+    exclude 'reference/cat/plugins.asciidoc'
+  }
 }
 
 listSnippets.docs = buildRestTests.docs

modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java

@@ -117,6 +117,7 @@ public class ReindexRestClientSslTests extends ESTestCase {
     }
 
     public void testClientFailsWithUntrustedCertificate() throws IOException {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         final List<Thread> threads = new ArrayList<>();
         final Settings settings = Settings.builder()
             .put("path.home", createTempDir())

modules/transport-netty4/build.gradle

@@ -112,6 +112,7 @@ thirdPartyAudit {
     'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
     'org.bouncycastle.jce.provider.BouncyCastleProvider',
     'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+    'org.bouncycastle.asn1.x500.X500Name',
 
     // from io.netty.handler.ssl.JettyNpnSslEngine (netty)
     'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
@@ -168,7 +169,6 @@ thirdPartyAudit {
     'org.eclipse.jetty.alpn.ALPN$ServerProvider',
     'org.eclipse.jetty.alpn.ALPN',
 
-
     'org.conscrypt.AllocatedBuffer',
     'org.conscrypt.BufferAllocator',
     'org.conscrypt.Conscrypt',
@@ -196,12 +196,10 @@ thirdPartyAudit {
   )
 }
 
-rootProject.globalInfo.ready {
-  if (BuildParams.inFipsJvm == false) {
-    // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in
-    // a FIPS JVM with BouncyCastleFIPS Provider
-    thirdPartyAudit.ignoreMissingClasses(
-      'org.bouncycastle.asn1.x500.X500Name'
-    )
-  }
+if (BuildParams.inFipsJvm == false) {
+  // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in
+  // a FIPS JVM with BouncyCastleFIPS Provider
+  thirdPartyAudit.ignoreMissingClasses(
+    'org.bouncycastle.asn1.x500.X500Name'
+  )
 }

plugins/ingest-attachment/build.gradle

@@ -89,7 +89,7 @@ thirdPartyAudit {
 }
 
 thirdPartyAudit.onlyIf {
-  // FIPS JVM includes manny classes from bouncycastle which count as jar hell for the third party audit,
+  // FIPS JVM includes many classes from bouncycastle which count as jar hell for the third party audit,
   // rather than provide a long list of exclusions, disable the check on FIPS.
   BuildParams.inFipsJvm == false
 }

plugins/transport-nio/build.gradle

@@ -64,6 +64,7 @@ thirdPartyAudit {
     'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
     'org.bouncycastle.jce.provider.BouncyCastleProvider',
     'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+    'org.bouncycastle.asn1.x500.X500Name',
 
     // from io.netty.handler.ssl.JettyNpnSslEngine (netty)
     'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
@@ -154,12 +155,12 @@ thirdPartyAudit {
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator'
   )
 }
-rootProject.globalInfo.ready {
-  if (BuildParams.inFipsJvm == false) {
-    // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in
-    // a FIPS JVM with BouncyCastleFIPS Provider
-    thirdPartyAudit.ignoreMissingClasses(
-      'org.bouncycastle.asn1.x500.X500Name'
-    )
-  }
+
+if (BuildParams.inFipsJvm == false) {
+  // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in
+  // a FIPS JVM with BouncyCastleFIPS Provider
+  thirdPartyAudit.ignoreMissingClasses(
+    'org.bouncycastle.asn1.x500.X500Name'
+  )
 }
+

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java

@@ -578,11 +578,12 @@ public class SSLServiceTests extends ESTestCase {
 
         final SSLService sslService = new SSLService(settings, env);
         final List<CertificateInfo> certificates = new ArrayList<>(sslService.getLoadedCertificates());
-        assertThat(certificates, iterableWithSize(10));
+        assertThat(certificates, iterableWithSize(13));
         Collections.sort(certificates,
             Comparator.comparing((CertificateInfo c) -> c.alias() == null ? "" : c.alias()).thenComparing(CertificateInfo::path));
 
         final Iterator<CertificateInfo> iterator = certificates.iterator();
+
         CertificateInfo cert = iterator.next();
         assertThat(cert.alias(), nullValue());
         assertThat(cert.path(), equalTo(pemPath.toString()));
@@ -646,6 +647,15 @@ public class SSLServiceTests extends ESTestCase {
         assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2045-10-02T09:43:18.000Z")));
         assertThat(cert.hasPrivateKey(), equalTo(true));
 
+        cert = iterator.next();
+        assertThat(cert.alias(), equalTo("testnode_dsa"));
+        assertThat(cert.path(), equalTo(p12Path.toString()));
+        assertThat(cert.format(), equalTo("PKCS12"));
+        assertThat(cert.serialNumber(), equalTo("223c736a"));
+        assertThat(cert.subjectDn(), equalTo("CN=Elasticsearch Test Node"));
+        assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2045-10-02T09:43:18.000Z")));
+        assertThat(cert.hasPrivateKey(), equalTo(true));
+
         cert = iterator.next();
         assertThat(cert.alias(), equalTo("testnode_ec"));
         assertThat(cert.path(), equalTo(jksPath.toString()));
@@ -655,6 +665,15 @@ public class SSLServiceTests extends ESTestCase {
         assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2045-10-02T09:36:10.000Z")));
         assertThat(cert.hasPrivateKey(), equalTo(true));
 
+        cert = iterator.next();
+        assertThat(cert.alias(), equalTo("testnode_ec"));
+        assertThat(cert.path(), equalTo(p12Path.toString()));
+        assertThat(cert.format(), equalTo("PKCS12"));
+        assertThat(cert.serialNumber(), equalTo("7268203b"));
+        assertThat(cert.subjectDn(), equalTo("CN=Elasticsearch Test Node"));
+        assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2045-10-02T09:36:10.000Z")));
+        assertThat(cert.hasPrivateKey(), equalTo(true));
+
         cert = iterator.next();
         assertThat(cert.alias(), equalTo("testnode_rsa"));
         assertThat(cert.path(), equalTo(jksPath.toString()));
@@ -670,9 +689,18 @@ public class SSLServiceTests extends ESTestCase {
         assertThat(cert.format(), equalTo("PKCS12"));
         assertThat(cert.serialNumber(), equalTo("b8b96c37e332cccb"));
         assertThat(cert.subjectDn(), equalTo("CN=Elasticsearch Test Node, OU=elasticsearch, O=org"));
-        assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2019-09-22T18:52:57Z")));
+        assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2019-09-22T18:52:57.000Z")));
         assertThat(cert.hasPrivateKey(), equalTo(true));
 
+        cert = iterator.next();
+        assertThat(cert.alias(), equalTo("trusted_testnode_ec"));
+        assertThat(cert.path(), equalTo(jksPath.toString()));
+        assertThat(cert.format(), equalTo("jks"));
+        assertThat(cert.serialNumber(), equalTo("7268203b"));
+        assertThat(cert.subjectDn(), equalTo("CN=Elasticsearch Test Node"));
+        assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2045-10-02T09:36:10.000Z")));
+        assertThat(cert.hasPrivateKey(), equalTo(false));
+
         assertFalse(iterator.hasNext());
     }
 

x-pack/plugin/security/cli/build.gradle

@@ -19,18 +19,16 @@ dependencyLicenses {
   mapping from: /bc.*/, to: 'bouncycastle'
 }
 
-rootProject.globalInfo.ready {
-  if (BuildParams.inFipsJvm) {
-    test.enabled = false
-    testingConventions.enabled = false
-    // Forbiden APIs non-portable checks fail because bouncy castle classes being used from the FIPS JDK since those are
-    // not part of the Java specification - all of this is as designed, so we have to relax this check for FIPS.
-    tasks.withType(CheckForbiddenApis) {
-      bundledSignatures -= "jdk-non-portable"
-    }
-    // FIPS JVM includes many classes from bouncycastle which count as jar hell for the third party audit,
-    // rather than provide a long list of exclusions, disable the check on FIPS.
-    thirdPartyAudit.enabled = false
-
+if (BuildParams.inFipsJvm) {
+  test.enabled = false
+  testingConventions.enabled = false
+  // Forbiden APIs non-portable checks fail because bouncy castle classes being used from the FIPS JDK since those are
+  // not part of the Java specification - all of this is as designed, so we have to relax this check for FIPS.
+  tasks.withType(CheckForbiddenApis) {
+    bundledSignatures -= "jdk-non-portable"
   }
+  // FIPS JVM includes many classes from bouncycastle which count as jar hell for the third party audit,
+  // rather than provide a long list of exclusions, disable the check on FIPS.
+  thirdPartyAudit.enabled = false
+
 }