Upgrade SDK and test discovery-ec2 credential providers (#41732)

Upgrades the AWS SDK to the same version that we're using for the repository-s3 plugin, providing
testing capabilities to override certain SDK endpoints in order to point them to localhost for testing.
Adds tests for the various credential providers.

plugins/discovery-ec2/build.gradle

@@ -23,7 +23,7 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.11.187'
+  'aws': '1.11.505'
 ]
 
 dependencies {

plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.187.jar.sha1

@@ -1 +0,0 @@
-6f47fcd3c2917bef69dc36aba203c5ea4af9bf24

plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.505.jar.sha1

@@ -0,0 +1 @@
+d19328c227b2b5ad81d137361ebc9cbcd0396465

plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.187.jar.sha1

@@ -1 +0,0 @@
-f3e5a8601f3105624674b1a12ca34f453a4b5895

plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.505.jar.sha1

@@ -0,0 +1 @@
+b669b3c90ea9bf73734ab26f0cb30c5c66addf55

plugins/discovery-ec2/qa/amazon-ec2/build.gradle

@@ -20,6 +20,7 @@
 
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.AntFixture
+import org.elasticsearch.gradle.test.RestIntegTestTask
 
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
@@ -30,14 +31,6 @@ dependencies {
 
 final int ec2NumberOfNodes = 3
 
-/** A task to start the AmazonEC2Fixture which emulates an EC2 service **/
-task ec2Fixture(type: AntFixture) {
-    dependsOn compileTestJava
-    env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
-    executable = new File(project.runtimeJavaHome, 'bin/java')
-    args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, "${buildDir}/testclusters/integTest-1/config/unicast_hosts.txt"
-}
-
 Map<String, Object> expansions = [
     'expected_nodes': ec2NumberOfNodes
 ]
@@ -47,20 +40,71 @@ processTestResources {
     MavenFilteringHack.filter(it, expansions)
 }
 
-integTest {
-    dependsOn ec2Fixture, project(':plugins:discovery-ec2').bundlePlugin
-}
+// disable default test task, use spezialized ones below
+integTest.enabled = false
+
+/*
+ * Test using various credential providers (see also https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/credentials.html):
+ * - Elasticsearch Keystore (secure settings discovery.ec2.access_key and discovery.ec2.secret_key)
+ * - Java system properties (aws.accessKeyId and aws.secretAccessKey)
+ * - Environment variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
+ * - ECS container credentials (loaded from ECS if the environment variable AWS_CONTAINER_CREDENTIALS_RELATIVE_URI is set)
+ * - Instance profile credentials (delivered through the EC2 metadata service)
+ *
+ * Notably missing is a test for the default credential profiles file, which is located at ~/.aws/credentials and would at least require a
+ * custom Java security policy to work.
+ */
+['KeyStore', 'EnvVariables', 'SystemProperties', 'ContainerCredentials', 'InstanceProfile'].forEach { action ->
+    AntFixture fixture = tasks.create(name: "ec2Fixture${action}", type: AntFixture) {
+        dependsOn compileTestJava
+        env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
+        executable = new File(project.runtimeJavaHome, 'bin/java')
+        args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, "${buildDir}/testclusters/integTest${action}-1/config/unicast_hosts.txt"
+    }
+
+    tasks.create(name: "integTest${action}", type: RestIntegTestTask) {
+        dependsOn fixture, project(':plugins:discovery-ec2').bundlePlugin
+    }
+
+    check.dependsOn("integTest${action}")
+
+    testClusters."integTest${action}" {
+        numberOfNodes = ec2NumberOfNodes
+        plugin file(project(':plugins:discovery-ec2').bundlePlugin.archiveFile)
 
-testClusters.integTest {
-    numberOfNodes = ec2NumberOfNodes
-    plugin file(project(':plugins:discovery-ec2').bundlePlugin.archiveFile)
+        setting 'discovery.seed_providers', 'ec2'
+        setting 'network.host', '_ec2_'
+        setting 'discovery.ec2.endpoint', { "http://${-> fixture.addressAndPort}" }
 
+        systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> fixture.addressAndPort}" }
+    }
+}
+
+// Extra config for KeyStore
+testClusters.integTestKeyStore {
     keystore 'discovery.ec2.access_key', 'ec2_integration_test_access_key'
     keystore 'discovery.ec2.secret_key', 'ec2_integration_test_secret_key'
-    
-    setting 'discovery.seed_providers', 'ec2'
-    setting 'network.host', '_ec2_'
-    setting 'discovery.ec2.endpoint', { "http://${ec2Fixture.addressAndPort}" }
-    
-    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${ec2Fixture.addressAndPort}" }
 }
+
+// Extra config for EnvVariables
+testClusters.integTestEnvVariables {
+    environment 'AWS_ACCESS_KEY_ID', 'ec2_integration_test_access_key'
+    environment 'AWS_SECRET_ACCESS_KEY', 'ec2_integration_test_secret_key'
+}
+
+// Extra config for SystemProperties
+testClusters.integTestSystemProperties {
+    systemProperty 'aws.accessKeyId', 'ec2_integration_test_access_key'
+    systemProperty 'aws.secretKey', 'ec2_integration_test_secret_key'
+}
+
+// Extra config for ContainerCredentials
+ec2FixtureContainerCredentials.env 'ACTIVATE_CONTAINER_CREDENTIALS', true
+
+testClusters.integTestContainerCredentials {
+    environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
+        { "http://${-> tasks.findByName("ec2FixtureContainerCredentials").addressAndPort}/ecs_credentials_endpoint" }
+}
+
+// Extra config for InstanceProfile
+ec2FixtureInstanceProfile.env 'ACTIVATE_INSTANCE_PROFILE', true

plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java

@@ -18,10 +18,12 @@
  */
 package org.elasticsearch.discovery.ec2;
 
+import com.amazonaws.util.DateUtils;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.utils.URLEncodedUtils;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.fixture.AbstractHttpFixture;
@@ -34,8 +36,12 @@ import java.io.StringWriter;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
@@ -45,10 +51,14 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 public class AmazonEC2Fixture extends AbstractHttpFixture {
 
     private final Path nodes;
+    private final boolean instanceProfile;
+    private final boolean containerCredentials;
 
-    private AmazonEC2Fixture(final String workingDir, final String nodesUriPath) {
+    private AmazonEC2Fixture(final String workingDir, final String nodesUriPath, boolean instanceProfile, boolean containerCredentials) {
         super(workingDir);
         this.nodes = toPath(Objects.requireNonNull(nodesUriPath));
+        this.instanceProfile = instanceProfile;
+        this.containerCredentials = containerCredentials;
     }
 
     public static void main(String[] args) throws Exception {
@@ -56,7 +66,10 @@ public class AmazonEC2Fixture extends AbstractHttpFixture {
             throw new IllegalArgumentException("AmazonEC2Fixture <working directory> <nodes transport uri file>");
         }
 
-        final AmazonEC2Fixture fixture = new AmazonEC2Fixture(args[0], args[1]);
+        boolean instanceProfile = Booleans.parseBoolean(System.getenv("ACTIVATE_INSTANCE_PROFILE"), false);
+        boolean containerCredentials = Booleans.parseBoolean(System.getenv("ACTIVATE_CONTAINER_CREDENTIALS"), false);
+
+        final AmazonEC2Fixture fixture = new AmazonEC2Fixture(args[0], args[1], instanceProfile, containerCredentials);
         fixture.listen();
     }
 
@@ -65,6 +78,12 @@ public class AmazonEC2Fixture extends AbstractHttpFixture {
         if ("/".equals(request.getPath()) && (HttpPost.METHOD_NAME.equals(request.getMethod()))) {
             final String userAgent = request.getHeader("User-Agent");
             if (userAgent != null && userAgent.startsWith("aws-sdk-java")) {
+
+                final String auth = request.getHeader("Authorization");
+                if (auth == null || auth.contains("ec2_integration_test_access_key") == false) {
+                    throw new IllegalArgumentException("wrong access key: " + auth);
+                }
+
                 // Simulate an EC2 DescribeInstancesResponse
                 byte[] responseBody = EMPTY_BYTE;
                 for (NameValuePair parse : URLEncodedUtils.parse(new String(request.getBody(), UTF_8), UTF_8)) {
@@ -79,6 +98,32 @@ public class AmazonEC2Fixture extends AbstractHttpFixture {
         if ("/latest/meta-data/local-ipv4".equals(request.getPath()) && (HttpGet.METHOD_NAME.equals(request.getMethod()))) {
             return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, "127.0.0.1".getBytes(UTF_8));
         }
+
+        if (instanceProfile &&
+            "/latest/meta-data/iam/security-credentials/".equals(request.getPath()) &&
+            HttpGet.METHOD_NAME.equals(request.getMethod())) {
+            final Map<String, String> headers = new HashMap<>(contentType("text/plain"));
+            return new Response(RestStatus.OK.getStatus(), headers, "my_iam_profile".getBytes(UTF_8));
+        }
+
+        if ((containerCredentials &&
+            "/ecs_credentials_endpoint".equals(request.getPath()) &&
+            HttpGet.METHOD_NAME.equals(request.getMethod())) ||
+            ("/latest/meta-data/iam/security-credentials/my_iam_profile".equals(request.getPath()) &&
+            HttpGet.METHOD_NAME.equals(request.getMethod()))) {
+            final Date expiration = new Date(new Date().getTime() + TimeUnit.DAYS.toMillis(1));
+            final String response = "{"
+                + "\"AccessKeyId\": \"" + "ec2_integration_test_access_key" + "\","
+                + "\"Expiration\": \"" + DateUtils.formatISO8601Date(expiration) + "\","
+                + "\"RoleArn\": \"" + "test" + "\","
+                + "\"SecretAccessKey\": \"" + "test" + "\","
+                + "\"Token\": \"" + "test" + "\""
+                + "}";
+
+            final Map<String, String> headers = new HashMap<>(contentType("application/json"));
+            return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8));
+        }
+
         return null;
     }
 

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java

@@ -22,13 +22,12 @@ package org.elasticsearch.discovery.ec2;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
 import com.amazonaws.http.IdleConnectionReaper;
-import com.amazonaws.internal.StaticCredentialsProvider;
 import com.amazonaws.retry.RetryPolicy;
 import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.AmazonEC2Client;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
@@ -97,11 +96,11 @@ class AwsEc2ServiceImpl implements AwsEc2Service {
     static AWSCredentialsProvider buildCredentials(Logger logger, Ec2ClientSettings clientSettings) {
         final AWSCredentials credentials = clientSettings.credentials;
         if (credentials == null) {
-            logger.debug("Using either environment variables, system properties or instance profile credentials");
-            return new DefaultAWSCredentialsProviderChain();
+            logger.debug("Using default provider chain");
+            return DefaultAWSCredentialsProviderChain.getInstance();
         } else {
             logger.debug("Using basic key/secret credentials");
-            return new StaticCredentialsProvider(credentials);
+            return new AWSStaticCredentialsProvider(credentials);
         }
     }