Upgrade to AWS SDK 1.11.749 (#53962)

Upgrading AWS SDK to v1.11.749.
Required building clients inside privileged contexts because some class loading that requires privileges now happens there and working around a new SDK bug in the S3 client builder.

Closes #53191

plugins/discovery-ec2/build.gradle

@@ -25,7 +25,7 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.11.636'
+  'aws': '1.11.749'
 ]
 
 dependencies {

plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.636.jar.sha1

@@ -1 +0,0 @@
-84c9f180f8f60f6f1433c9c5253fcb704593b121

plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.749.jar.sha1

@@ -0,0 +1 @@
+1da5c1549295cfeebc67fc1c7539785a9441755b

plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.636.jar.sha1

@@ -1 +0,0 @@
-d32fc4ae314dbee9717302a3119cba0f735c04b1

plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.749.jar.sha1

@@ -0,0 +1 @@
+0865e0937c6500acf62ce9c8964eac76a8718f5f

plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java

@@ -22,6 +22,7 @@ import com.amazonaws.util.DateUtils;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.utils.URLEncodedUtils;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.SuppressForbidden;
@@ -106,6 +107,13 @@ public class AmazonEC2Fixture extends AbstractHttpFixture {
             return new Response(RestStatus.OK.getStatus(), headers, "my_iam_profile".getBytes(UTF_8));
         }
 
+        if (instanceProfile && "/latest/api/token".equals(request.getPath())
+            && HttpPut.METHOD_NAME.equals(request.getMethod())) {
+            // TODO: Implement IMDSv2 behavior here. For now this just returns a 403 which makes the SDK fall back to IMDSv1
+            //       which is implemented in this fixture
+            return new Response(RestStatus.FORBIDDEN.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
+        }
+
         if ((containerCredentials &&
             "/ecs_credentials_endpoint".equals(request.getPath()) &&
             HttpGet.METHOD_NAME.equals(request.getMethod())) ||

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java

@@ -24,9 +24,10 @@ import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.http.IdleConnectionReaper;
 import com.amazonaws.services.ec2.AmazonEC2;
-import com.amazonaws.services.ec2.AmazonEC2Client;
+import com.amazonaws.services.ec2.AmazonEC2ClientBuilder;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
@@ -45,17 +46,18 @@ class AwsEc2ServiceImpl implements AwsEc2Service {
     private AmazonEC2 buildClient(Ec2ClientSettings clientSettings) {
         final AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings);
         final ClientConfiguration configuration = buildConfiguration(clientSettings);
-        final AmazonEC2 client = buildClient(credentials, configuration);
-        if (Strings.hasText(clientSettings.endpoint)) {
-            logger.debug("using explicit ec2 endpoint [{}]", clientSettings.endpoint);
-            client.setEndpoint(clientSettings.endpoint);
-        }
-        return client;
+        return buildClient(credentials, configuration, clientSettings.endpoint);
     }
 
     // proxy for testing
-    AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
-        return new AmazonEC2Client(credentials, configuration);
+    AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration, String endpoint) {
+        final AmazonEC2ClientBuilder builder = AmazonEC2ClientBuilder.standard().withCredentials(credentials)
+            .withClientConfiguration(configuration);
+        if (Strings.hasText(endpoint)) {
+            logger.debug("using explicit ec2 endpoint [{}]", endpoint);
+            builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));
+        }
+        return SocketAccess.doPrivileged(builder::build);
     }
 
     // pkg private for tests

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java

@@ -39,7 +39,6 @@ import java.util.Arrays;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
 
 public class Ec2DiscoveryPluginTests extends ESTestCase {
 
@@ -96,7 +95,7 @@ public class Ec2DiscoveryPluginTests extends ESTestCase {
     public void testDefaultEndpoint() throws IOException {
         try (Ec2DiscoveryPluginMock plugin = new Ec2DiscoveryPluginMock(Settings.EMPTY)) {
             final String endpoint = ((AmazonEC2Mock) plugin.ec2Service.client().client()).endpoint;
-            assertThat(endpoint, nullValue());
+            assertThat(endpoint, is(""));
         }
     }
 
@@ -199,8 +198,9 @@ public class Ec2DiscoveryPluginTests extends ESTestCase {
         Ec2DiscoveryPluginMock(Settings settings) {
             super(settings, new AwsEc2ServiceImpl() {
                 @Override
-                AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
-                    return new AmazonEC2Mock(credentials, configuration);
+                AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration,
+                                      String endpoint) {
+                    return new AmazonEC2Mock(credentials, configuration, endpoint);
                 }
             });
         }
@@ -212,13 +212,9 @@ public class Ec2DiscoveryPluginTests extends ESTestCase {
         final AWSCredentialsProvider credentials;
         final ClientConfiguration configuration;
 
-        AmazonEC2Mock(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
+        AmazonEC2Mock(AWSCredentialsProvider credentials, ClientConfiguration configuration, String endpoint) {
             this.credentials = credentials;
             this.configuration = configuration;
-        }
-
-        @Override
-        public void setEndpoint(String endpoint) throws IllegalArgumentException {
             this.endpoint = endpoint;
         }
 

plugins/repository-s3/build.gradle

@@ -30,7 +30,7 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.11.636'
+  'aws': '1.11.749'
 ]
 
 dependencies {

plugins/repository-s3/licenses/aws-java-sdk-core-1.11.636.jar.sha1

@@ -1 +0,0 @@
-84c9f180f8f60f6f1433c9c5253fcb704593b121

plugins/repository-s3/licenses/aws-java-sdk-core-1.11.749.jar.sha1

@@ -0,0 +1 @@
+1da5c1549295cfeebc67fc1c7539785a9441755b

plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.636.jar.sha1

@@ -1 +0,0 @@
-f86fc1993ac8122f6f02a8eb9b467b5f945cd76b

plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.749.jar.sha1

@@ -0,0 +1 @@
+7d069f82723907ccdbd0c91ef0ac76046f5c9652

plugins/repository-s3/licenses/jmespath-java-1.11.636.jar.sha1

@@ -1 +0,0 @@
-e468c349ce410171a1d5df7fa0fa377d52c5d651

plugins/repository-s3/licenses/jmespath-java-1.11.749.jar.sha1

@@ -0,0 +1 @@
+778866bc557dba508ee0eab2a0c5bfde468e49e6

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -141,7 +141,12 @@ class S3Service implements Closeable {
         builder.withCredentials(buildCredentials(logger, clientSettings));
         builder.withClientConfiguration(buildConfiguration(clientSettings));
 
-        final String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME;
+        String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME;
+        if ((endpoint.startsWith("http://") || endpoint.startsWith("https://")) == false) {
+            // Manually add the schema to the endpoint to work around https://github.com/aws/aws-sdk-java/issues/2274
+            // TODO: Remove this once fixed in the AWS SDK
+            endpoint = clientSettings.protocol.toString() + "://" + endpoint;
+        }
         final String region = Strings.hasLength(clientSettings.region) ? clientSettings.region : null;
         logger.debug("using endpoint [{}] and region [{}]", endpoint, region);
 
@@ -160,7 +165,7 @@ class S3Service implements Closeable {
         if (clientSettings.disableChunkedEncoding) {
             builder.disableChunkedEncoding();
         }
-        return builder.build();
+        return SocketAccess.doPrivileged(builder::build);
     }
 
     // pkg private for tests