Add support for AWS session tokens (#30414)

AWS supports the creation and use of credentials that are only valid for a
fixed period of time. These credentials comprise three parts: the usual access
key and secret key, together with a session token. This commit adds support for
these three-part credentials to the EC2 discovery plugin and the S3 repository
plugin.

Note that session tokens are only valid for a limited period of time and yet
there is no mechanism for refreshing or rotating them when they expire without
restarting Elasticsearch.  Nonetheless, this feature is already useful for
nodes that need only run for a few days, such as for training, testing or
evaluation. #29135 tracks the work towards allowing these credentials to be
refreshed at runtime.

Resolves #16428

docs/plugins/discovery-ec2.asciidoc

@@ -40,11 +40,15 @@ Those that must be stored in the keystore are marked as `Secure`.
 
 `access_key`::
 
-    An s3 access key. The `secret_key` setting must also be specified. (Secure)
+    An ec2 access key. The `secret_key` setting must also be specified. (Secure)
 
 `secret_key`::
 
-    An s3 secret key. The `access_key` setting must also be specified. (Secure)
+    An ec2 secret key. The `access_key` setting must also be specified. (Secure)
+
+`session_token`::
+    An ec2 session token. The `access_key` and `secret_key` settings must also
+    be specified. (Secure)
 
 `endpoint`::
 

docs/plugins/repository-s3.asciidoc

@@ -73,6 +73,10 @@ are marked as `Secure`.
 
     An s3 secret key. The `access_key` setting must also be specified. (Secure)
 
+`session_token`::
+    An s3 session token. The `access_key` and `secret_key` settings must also
+    be specified. (Secure)
+
 `endpoint`::
 
     The s3 service endpoint to connect to. This will be automatically

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java

@@ -19,12 +19,9 @@
 
 package org.elasticsearch.discovery.ec2;
 
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicReference;
-
 import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
 import com.amazonaws.http.IdleConnectionReaper;
 import com.amazonaws.internal.StaticCredentialsProvider;
@@ -39,6 +36,9 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.LazyInitializable;
 
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicReference;
+
 class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service {
 
     public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/";
@@ -99,7 +99,7 @@ class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service {
 
     // pkg private for tests
     static AWSCredentialsProvider buildCredentials(Logger logger, Ec2ClientSettings clientSettings) {
-        final BasicAWSCredentials credentials = clientSettings.credentials;
+        final AWSCredentials credentials = clientSettings.credentials;
         if (credentials == null) {
             logger.debug("Using either environment variables, system properties or instance profile credentials");
             return new DefaultAWSCredentialsProviderChain();

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java

@@ -21,14 +21,20 @@ package org.elasticsearch.discovery.ec2;
 
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.BasicAWSCredentials;
-
+import com.amazonaws.auth.BasicSessionCredentials;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.TimeValue;
+
 import java.util.Locale;
 
 /**
@@ -42,6 +48,9 @@ final class Ec2ClientSettings {
     /** The secret key (ie password) for connecting to ec2. */
     static final Setting<SecureString> SECRET_KEY_SETTING = SecureSetting.secureString("discovery.ec2.secret_key", null);
 
+    /** The session token for connecting to ec2. */
+    static final Setting<SecureString> SESSION_TOKEN_SETTING = SecureSetting.secureString("discovery.ec2.session_token", null);
+
     /** The host name of a proxy to connect to ec2 through. */
     static final Setting<String> PROXY_HOST_SETTING = Setting.simpleString("discovery.ec2.proxy.host", Property.NodeScope);
 
@@ -66,8 +75,12 @@ final class Ec2ClientSettings {
     static final Setting<TimeValue> READ_TIMEOUT_SETTING = Setting.timeSetting("discovery.ec2.read_timeout",
             TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope);
 
+    private static final Logger logger = Loggers.getLogger(Ec2ClientSettings.class);
+
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger);
+
     /** Credentials to authenticate with ec2. */
-    final BasicAWSCredentials credentials;
+    final AWSCredentials credentials;
 
     /**
      * The ec2 endpoint the client should talk to, or empty string to use the
@@ -96,7 +109,7 @@ final class Ec2ClientSettings {
     /** The read timeout for the ec2 client. */
     final int readTimeoutMillis;
 
-    protected Ec2ClientSettings(BasicAWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort,
+    protected Ec2ClientSettings(AWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort,
             String proxyUsername, String proxyPassword, int readTimeoutMillis) {
         this.credentials = credentials;
         this.endpoint = endpoint;
@@ -108,26 +121,45 @@ final class Ec2ClientSettings {
         this.readTimeoutMillis = readTimeoutMillis;
     }
 
-    static BasicAWSCredentials loadCredentials(Settings settings) {
-        try (SecureString accessKey = ACCESS_KEY_SETTING.get(settings);
-                SecureString secretKey = SECRET_KEY_SETTING.get(settings);) {
-            if (accessKey.length() != 0) {
-                if (secretKey.length() != 0) {
-                    return new BasicAWSCredentials(accessKey.toString(), secretKey.toString());
+    static AWSCredentials loadCredentials(Settings settings) {
+        try (SecureString key = ACCESS_KEY_SETTING.get(settings);
+             SecureString secret = SECRET_KEY_SETTING.get(settings);
+             SecureString sessionToken = SESSION_TOKEN_SETTING.get(settings)) {
+            if (key.length() == 0 && secret.length() == 0) {
+                if (sessionToken.length() > 0) {
+                    throw new SettingsException("Setting [{}] is set but [{}] and [{}] are not",
+                        SESSION_TOKEN_SETTING.getKey(), ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey());
+                }
+
+                logger.debug("Using either environment variables, system properties or instance profile credentials");
+                return null;
+            } else {
+                if (key.length() == 0) {
+                    DEPRECATION_LOGGER.deprecated("Setting [{}] is set but [{}] is not, which will be unsupported in future",
+                        SECRET_KEY_SETTING.getKey(), ACCESS_KEY_SETTING.getKey());
+                }
+                if (secret.length() == 0) {
+                    DEPRECATION_LOGGER.deprecated("Setting [{}] is set but [{}] is not, which will be unsupported in future",
+                        ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey());
+                }
+
+                final AWSCredentials credentials;
+                if (sessionToken.length() == 0) {
+                    logger.debug("Using basic key/secret credentials");
+                    credentials = new BasicAWSCredentials(key.toString(), secret.toString());
                 } else {
-                    throw new IllegalArgumentException("Missing secret key for ec2 client.");
+                    logger.debug("Using basic session credentials");
+                    credentials = new BasicSessionCredentials(key.toString(), secret.toString(), sessionToken.toString());
                 }
-            } else if (secretKey.length() != 0) {
-                throw new IllegalArgumentException("Missing access key for ec2 client.");
+                return credentials;
             }
-            return null;
         }
     }
 
     // pkg private for tests
     /** Parse settings for a single client. */
     static Ec2ClientSettings getClientSettings(Settings settings) {
-        final BasicAWSCredentials credentials = loadCredentials(settings);
+        final AWSCredentials credentials = loadCredentials(settings);
         try (SecureString proxyUsername = PROXY_USERNAME_SETTING.get(settings);
              SecureString proxyPassword = PROXY_PASSWORD_SETTING.get(settings)) {
             return new Ec2ClientSettings(

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java

@@ -106,6 +106,7 @@ public class Ec2DiscoveryPlugin extends Plugin implements DiscoveryPlugin, Reloa
         // Register EC2 discovery settings: discovery.ec2
         Ec2ClientSettings.ACCESS_KEY_SETTING,
         Ec2ClientSettings.SECRET_KEY_SETTING,
+        Ec2ClientSettings.SESSION_TOKEN_SETTING,
         Ec2ClientSettings.ENDPOINT_SETTING,
         Ec2ClientSettings.PROTOCOL_SETTING,
         Ec2ClientSettings.PROXY_HOST_SETTING,

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java

@@ -23,10 +23,11 @@ import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.BasicSessionCredentials;
 import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.ec2.AwsEc2ServiceImpl;
+import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.instanceOf;
@@ -44,15 +45,53 @@ public class AwsEc2ServiceImplTests extends ESTestCase {
         final MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("discovery.ec2.access_key", "aws_key");
         secureSettings.setString("discovery.ec2.secret_key", "aws_secret");
-        final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(settings, "aws_key", "aws_secret");
+        final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger,
+            Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
+        assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
+        assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
     }
 
-    protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings settings, String expectedKey, String expectedSecret) {
-        final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, Ec2ClientSettings.getClientSettings(settings))
-                .getCredentials();
-        assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
-        assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
+    public void testAWSSessionCredentialsWithElasticsearchAwsSettings() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("discovery.ec2.access_key", "aws_key");
+        secureSettings.setString("discovery.ec2.secret_key", "aws_secret");
+        secureSettings.setString("discovery.ec2.session_token", "aws_session_token");
+        final BasicSessionCredentials credentials = (BasicSessionCredentials) AwsEc2ServiceImpl.buildCredentials(logger,
+            Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
+        assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
+        assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
+        assertThat(credentials.getSessionToken(), is("aws_session_token"));
+    }
+
+    public void testDeprecationOfLoneAccessKey() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("discovery.ec2.access_key", "aws_key");
+        final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger,
+            Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
+        assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
+        assertThat(credentials.getAWSSecretKey(), is(""));
+        assertSettingDeprecationsAndWarnings(new String[]{},
+            "Setting [discovery.ec2.access_key] is set but [discovery.ec2.secret_key] is not, which will be unsupported in future");
+    }
+
+    public void testDeprecationOfLoneSecretKey() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("discovery.ec2.secret_key", "aws_secret");
+        final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger,
+            Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
+        assertThat(credentials.getAWSAccessKeyId(), is(""));
+        assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
+        assertSettingDeprecationsAndWarnings(new String[]{},
+            "Setting [discovery.ec2.secret_key] is set but [discovery.ec2.access_key] is not, which will be unsupported in future");
+    }
+
+    public void testRejectionOfLoneSessionToken() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("discovery.ec2.session_token", "aws_session_token");
+        SettingsException e = expectThrows(SettingsException.class, () -> AwsEc2ServiceImpl.buildCredentials(logger,
+            Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())));
+        assertThat(e.getMessage(), is(
+            "Setting [discovery.ec2.session_token] is set but [discovery.ec2.access_key] and [discovery.ec2.secret_key] are not"));
     }
 
     public void testAWSDefaultConfiguration() {

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java

@@ -19,22 +19,24 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.BasicSessionCredentials;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.test.ESTestCase;
+
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
 
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
-import org.elasticsearch.discovery.ec2.AwsEc2Service;
-import org.elasticsearch.common.settings.MockSecureSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.test.ESTestCase;
-
 public class Ec2DiscoveryPluginTests extends ESTestCase {
 
     private Settings getNodeAttributes(Settings settings, String url) {
@@ -106,6 +108,10 @@ public class Ec2DiscoveryPluginTests extends ESTestCase {
         final MockSecureSettings mockSecure1 = new MockSecureSettings();
         mockSecure1.setString(Ec2ClientSettings.ACCESS_KEY_SETTING.getKey(), "ec2_access_1");
         mockSecure1.setString(Ec2ClientSettings.SECRET_KEY_SETTING.getKey(), "ec2_secret_1");
+        final boolean mockSecure1HasSessionToken = randomBoolean();
+        if (mockSecure1HasSessionToken) {
+            mockSecure1.setString(Ec2ClientSettings.SESSION_TOKEN_SETTING.getKey(), "ec2_session_token_1");
+        }
         mockSecure1.setString(Ec2ClientSettings.PROXY_USERNAME_SETTING.getKey(), "proxy_username_1");
         mockSecure1.setString(Ec2ClientSettings.PROXY_PASSWORD_SETTING.getKey(), "proxy_password_1");
         final Settings settings1 = Settings.builder()
@@ -117,6 +123,10 @@ public class Ec2DiscoveryPluginTests extends ESTestCase {
         final MockSecureSettings mockSecure2 = new MockSecureSettings();
         mockSecure2.setString(Ec2ClientSettings.ACCESS_KEY_SETTING.getKey(), "ec2_access_2");
         mockSecure2.setString(Ec2ClientSettings.SECRET_KEY_SETTING.getKey(), "ec2_secret_2");
+        final boolean mockSecure2HasSessionToken = randomBoolean();
+        if (mockSecure2HasSessionToken) {
+            mockSecure2.setString(Ec2ClientSettings.SESSION_TOKEN_SETTING.getKey(), "ec2_session_token_2");
+        }
         mockSecure2.setString(Ec2ClientSettings.PROXY_USERNAME_SETTING.getKey(), "proxy_username_2");
         mockSecure2.setString(Ec2ClientSettings.PROXY_PASSWORD_SETTING.getKey(), "proxy_password_2");
         final Settings settings2 = Settings.builder()
@@ -127,27 +137,50 @@ public class Ec2DiscoveryPluginTests extends ESTestCase {
                 .build();
         try (Ec2DiscoveryPluginMock plugin = new Ec2DiscoveryPluginMock(settings1)) {
             try (AmazonEc2Reference clientReference = plugin.ec2Service.client()) {
-                assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSAccessKeyId(), is("ec2_access_1"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSSecretKey(), is("ec2_secret_1"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881));
-                assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1"));
+                {
+                    final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials();
+                    assertThat(credentials.getAWSAccessKeyId(), is("ec2_access_1"));
+                    assertThat(credentials.getAWSSecretKey(), is("ec2_secret_1"));
+                    if (mockSecure1HasSessionToken) {
+                        assertThat(credentials, instanceOf(BasicSessionCredentials.class));
+                        assertThat(((BasicSessionCredentials)credentials).getSessionToken(), is("ec2_session_token_1"));
+                    } else {
+                        assertThat(credentials, instanceOf(BasicAWSCredentials.class));
+                    }
+                    assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1"));
+                    assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1"));
+                    assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1"));
+                    assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881));
+                    assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1"));
+                }
                 // reload secure settings2
                 plugin.reload(settings2);
                 // client is not released, it is still using the old settings
-                assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSAccessKeyId(), is("ec2_access_1"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSSecretKey(), is("ec2_secret_1"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881));
-                assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1"));
+                {
+                    final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials();
+                    if (mockSecure1HasSessionToken) {
+                        assertThat(credentials, instanceOf(BasicSessionCredentials.class));
+                        assertThat(((BasicSessionCredentials)credentials).getSessionToken(), is("ec2_session_token_1"));
+                    } else {
+                        assertThat(credentials, instanceOf(BasicAWSCredentials.class));
+                    }
+                    assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1"));
+                    assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1"));
+                    assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1"));
+                    assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881));
+                    assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1"));
+                }
             }
             try (AmazonEc2Reference clientReference = plugin.ec2Service.client()) {
-                assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSAccessKeyId(), is("ec2_access_2"));
-                assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSSecretKey(), is("ec2_secret_2"));
+                final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials();
+                assertThat(credentials.getAWSAccessKeyId(), is("ec2_access_2"));
+                assertThat(credentials.getAWSSecretKey(), is("ec2_secret_2"));
+                if (mockSecure2HasSessionToken) {
+                    assertThat(credentials, instanceOf(BasicSessionCredentials.class));
+                    assertThat(((BasicSessionCredentials)credentials).getSessionToken(), is("ec2_session_token_2"));
+                } else {
+                    assertThat(credentials, instanceOf(BasicAWSCredentials.class));
+                }
                 assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_2"));
                 assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_2"));
                 assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_2"));

plugins/repository-s3/qa/amazon-s3/build.gradle

@@ -31,47 +31,81 @@ integTestCluster {
     plugin ':plugins:repository-s3'
 }
 
+forbiddenApisTest {
+    // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
+    bundledSignatures -= 'jdk-non-portable'
+    bundledSignatures += 'jdk-internal'
+}
+
 boolean useFixture = false
 
-String s3AccessKey = System.getenv("amazon_s3_access_key")
-String s3SecretKey = System.getenv("amazon_s3_secret_key")
-String s3Bucket = System.getenv("amazon_s3_bucket")
-String s3BasePath = System.getenv("amazon_s3_base_path")
+// We test against two repositories, one which uses the usual two-part "permanent" credentials and
+// the other which uses three-part "temporary" or "session" credentials.
+
+String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
+String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
+String s3PermanentBucket = System.getenv("amazon_s3_bucket")
+String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
+
+String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
+String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
+String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
+String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
+String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
+
+// If all these variables are missing then we are testing against the internal fixture instead, which has the following
+// credentials hard-coded in.
+
+if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath
+ && !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
+
+    s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
+    s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
+    s3PermanentBucket = 'permanent_bucket_test'
+    s3PermanentBasePath = 'integration_test'
+
+    s3TemporaryAccessKey = 's3_integration_test_temporary_access_key'
+    s3TemporarySecretKey = 's3_integration_test_temporary_secret_key'
+    s3TemporaryBucket = 'temporary_bucket_test'
+    s3TemporaryBasePath = 'integration_test'
+    s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
 
-if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) {
-    s3AccessKey = 's3_integration_test_access_key'
-    s3SecretKey = 's3_integration_test_secret_key'
-    s3Bucket = 'bucket_test'
-    s3BasePath = 'integration_test'
     useFixture = true
 }
 
 /** A task to start the AmazonS3Fixture which emulates a S3 service **/
 task s3Fixture(type: AntFixture) {
-    dependsOn testClasses
+    dependsOn compileTestJava
     env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
     executable = new File(project.runtimeJavaHome, 'bin/java')
-    args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3Bucket
+    args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3PermanentBucket, s3TemporaryBucket
 }
 
 Map<String, Object> expansions = [
-        'bucket': s3Bucket,
-        'base_path': s3BasePath
+    'permanent_bucket': s3PermanentBucket,
+    'permanent_base_path': s3PermanentBasePath,
+    'temporary_bucket': s3TemporaryBucket,
+    'temporary_base_path': s3TemporaryBasePath
 ]
-
 processTestResources {
     inputs.properties(expansions)
     MavenFilteringHack.filter(it, expansions)
 }
 
 integTestCluster {
-    keystoreSetting 's3.client.integration_test.access_key', s3AccessKey
-    keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey
+    keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
+    keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
+
+    keystoreSetting 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
+    keystoreSetting 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
+    keystoreSetting 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
 
     if (useFixture) {
+        println "Using internal test service to test the repository-s3 plugin"
         dependsOn s3Fixture
         /* Use a closure on the string to delay evaluation until tests are executed */
-        setting 's3.client.integration_test.endpoint', "http://${-> s3Fixture.addressAndPort}"
+        setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}"
+        setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}"
     } else {
         println "Using an external service to test the repository-s3 plugin"
     }

plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java

@@ -52,13 +52,19 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
 
     /** Request handlers for the requests made by the S3 client **/
     private final PathTrie<RequestHandler> handlers;
+    private final String permanentBucketName;
+    private final String temporaryBucketName;
 
     /**
      * Creates a {@link AmazonS3Fixture}
      */
-    private AmazonS3Fixture(final String workingDir, final String bucket) {
+    private AmazonS3Fixture(final String workingDir, final String permanentBucketName, final String temporaryBucketName) {
         super(workingDir);
-        this.buckets.put(bucket, new Bucket(bucket));
+        this.permanentBucketName = permanentBucketName;
+        this.temporaryBucketName = temporaryBucketName;
+
+        this.buckets.put(permanentBucketName, new Bucket(permanentBucketName));
+        this.buckets.put(temporaryBucketName, new Bucket(temporaryBucketName));
         this.handlers = defaultHandlers(buckets);
     }
 
@@ -67,21 +73,47 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         final RequestHandler handler = handlers.retrieve(request.getMethod() + " " + request.getPath(), request.getParameters());
         if (handler != null) {
             final String authorization = request.getHeader("Authorization");
-            if (authorization == null
-                || (authorization.length() > 0 && authorization.contains("s3_integration_test_access_key") == false)) {
-                return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Access Denied", "");
+            final String permittedBucket;
+            if (authorization.contains("s3_integration_test_permanent_access_key")) {
+                final String sessionToken = request.getHeader("x-amz-security-token");
+                if (sessionToken != null) {
+                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", "");
+                }
+                permittedBucket = permanentBucketName;
+            } else if (authorization.contains("s3_integration_test_temporary_access_key")) {
+                final String sessionToken = request.getHeader("x-amz-security-token");
+                if (sessionToken == null) {
+                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", "");
+                }
+                if (sessionToken.equals("s3_integration_test_temporary_session_token") == false) {
+                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", "");
+                }
+                permittedBucket = temporaryBucketName;
+            } else {
+                return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", "");
+            }
+
+            if (handler != null) {
+                final String bucket = request.getParam("bucket");
+                if (bucket != null && permittedBucket.equals(bucket) == false) {
+                    // allow a null bucket to support bucket-free APIs
+                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", "");
+                }
+                return handler.handle(request);
+            } else {
+                return newInternalError(request.getId(), "No handler defined for request [" + request + "]");
             }
-            return handler.handle(request);
         }
         return null;
     }
 
     public static void main(final String[] args) throws Exception {
-        if (args == null || args.length != 2) {
-            throw new IllegalArgumentException("AmazonS3Fixture <working directory> <bucket>");
+        if (args == null || args.length != 3) {
+            throw new IllegalArgumentException(
+                "AmazonS3Fixture <working directory> <bucket for permanent creds> <bucket for temporary creds>");
         }
 
-        final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1]);
+        final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1], args[2]);
         fixture.listen();
     }
 

plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml → plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml

@@ -3,34 +3,35 @@
 ---
 setup:
 
-  # Register repository
+  # Register repository with permanent credentials
   - do:
       snapshot.create_repository:
-        repository: repository
+        repository: repository_permanent
         body:
           type: s3
           settings:
-            bucket: ${bucket}
-            client: integration_test
-            base_path: ${base_path}
+            bucket: ${permanent_bucket}
+            client: integration_test_permanent
+            base_path: ${permanent_base_path}
             canned_acl: private
             storage_class: standard
 
 ---
-"Snapshot/Restore with repository-s3":
+"Snapshot/Restore with repository-s3 using permanent credentials":
 
   # Get repository
   - do:
       snapshot.get_repository:
-        repository: repository
+        repository: repository_permanent
 
-  - match: { repository.settings.bucket : ${bucket} }
-  - match: { repository.settings.client : "integration_test" }
-  - match: { repository.settings.base_path : ${base_path} }
-  - match: { repository.settings.canned_acl : "private" }
-  - match: { repository.settings.storage_class : "standard" }
-  - is_false: repository.settings.access_key
-  - is_false: repository.settings.secret_key
+  - match: { repository_permanent.settings.bucket : ${permanent_bucket} }
+  - match: { repository_permanent.settings.client : "integration_test_permanent" }
+  - match: { repository_permanent.settings.base_path : ${permanent_base_path} }
+  - match: { repository_permanent.settings.canned_acl : "private" }
+  - match: { repository_permanent.settings.storage_class : "standard" }
+  - is_false: repository_permanent.settings.access_key
+  - is_false: repository_permanent.settings.secret_key
+  - is_false: repository_permanent.settings.session_token
 
   # Index documents
   - do:
@@ -62,7 +63,7 @@ setup:
   # Create a first snapshot
   - do:
       snapshot.create:
-        repository: repository
+        repository: repository_permanent
         snapshot: snapshot-one
         wait_for_completion: true
 
@@ -73,7 +74,7 @@ setup:
 
   - do:
       snapshot.status:
-        repository: repository
+        repository: repository_permanent
         snapshot: snapshot-one
 
   - is_true: snapshots
@@ -115,7 +116,7 @@ setup:
   # Create a second snapshot
   - do:
       snapshot.create:
-        repository: repository
+        repository: repository_permanent
         snapshot: snapshot-two
         wait_for_completion: true
 
@@ -125,7 +126,7 @@ setup:
 
   - do:
       snapshot.get:
-        repository: repository
+        repository: repository_permanent
         snapshot: snapshot-one,snapshot-two
 
   - is_true: snapshots
@@ -140,7 +141,7 @@ setup:
   # Restore the second snapshot
   - do:
       snapshot.restore:
-        repository: repository
+        repository: repository_permanent
         snapshot: snapshot-two
         wait_for_completion: true
 
@@ -158,7 +159,7 @@ setup:
   # Restore the first snapshot
   - do:
       snapshot.restore:
-        repository: repository
+        repository: repository_permanent
         snapshot: snapshot-one
         wait_for_completion: true
 
@@ -171,12 +172,12 @@ setup:
   # Remove the snapshots
   - do:
       snapshot.delete:
-        repository: repository
+        repository: repository_permanent
         snapshot: snapshot-two
 
   - do:
       snapshot.delete:
-        repository: repository
+        repository: repository_permanent
         snapshot: snapshot-one
 
 ---
@@ -185,7 +186,7 @@ setup:
   - do:
       catch: /repository_exception/
       snapshot.create_repository:
-        repository: repository
+        repository: repository_permanent
         body:
           type: s3
           settings:
@@ -198,11 +199,11 @@ setup:
   - do:
       catch: /repository_exception/
       snapshot.create_repository:
-        repository: repository
+        repository: repository_permanent
         body:
           type: s3
           settings:
-            bucket: repository
+            bucket: repository_permanent
             client: unknown
 
 ---
@@ -211,7 +212,7 @@ setup:
   - do:
       catch: /snapshot_missing_exception/
       snapshot.get:
-        repository: repository
+        repository: repository_permanent
         snapshot: missing
 
 ---
@@ -220,7 +221,7 @@ setup:
   - do:
       catch: /snapshot_missing_exception/
       snapshot.delete:
-        repository: repository
+        repository: repository_permanent
         snapshot: missing
 
 ---
@@ -229,7 +230,7 @@ setup:
   - do:
       catch: /snapshot_restore_exception/
       snapshot.restore:
-        repository: repository
+        repository: repository_permanent
         snapshot: missing
         wait_for_completion: true
 
@@ -239,4 +240,4 @@ teardown:
   # Remove our repository
   - do:
      snapshot.delete_repository:
-       repository: repository
+       repository: repository_permanent

plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml

@@ -0,0 +1,243 @@
+# Integration tests for repository-s3
+
+---
+setup:
+
+  # Register repository with temporary credentials
+  - do:
+      snapshot.create_repository:
+        repository: repository_temporary
+        body:
+          type: s3
+          settings:
+            bucket: ${temporary_bucket}
+            client: integration_test_temporary
+            base_path: ${temporary_base_path}
+            canned_acl: private
+            storage_class: standard
+
+---
+"Snapshot/Restore with repository-s3 using temporary credentials":
+
+  # Get repository
+  - do:
+      snapshot.get_repository:
+        repository: repository_temporary
+
+  - match: { repository_temporary.settings.bucket : ${temporary_bucket} }
+  - match: { repository_temporary.settings.client : "integration_test_temporary" }
+  - match: { repository_temporary.settings.base_path : ${temporary_base_path} }
+  - match: { repository_temporary.settings.canned_acl : "private" }
+  - match: { repository_temporary.settings.storage_class : "standard" }
+  - is_false: repository_temporary.settings.access_key
+  - is_false: repository_temporary.settings.secret_key
+  - is_false: repository_temporary.settings.session_token
+
+  # Index documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    1
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    2
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    3
+          - snapshot: one
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Create a first snapshot
+  - do:
+      snapshot.create:
+        repository: repository_temporary
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-one }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.include_global_state: true }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.status:
+        repository: repository_temporary
+        snapshot: snapshot-one
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: snapshot-one }
+  - match: { snapshots.0.state : SUCCESS }
+
+  # Index more documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    4
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    5
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    6
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    7
+          - snapshot: two
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Create a second snapshot
+  - do:
+      snapshot.create:
+        repository: repository_temporary
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-two }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.get:
+        repository: repository_temporary
+        snapshot: snapshot-one,snapshot-two
+
+  - is_true: snapshots
+  - match: { snapshots.0.state : SUCCESS }
+  - match: { snapshots.1.state : SUCCESS }
+
+  # Delete the index
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the second snapshot
+  - do:
+      snapshot.restore:
+        repository: repository_temporary
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Delete the index again
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the first snapshot
+  - do:
+      snapshot.restore:
+        repository: repository_temporary
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Remove the snapshots
+  - do:
+      snapshot.delete:
+        repository: repository_temporary
+        snapshot: snapshot-two
+
+  - do:
+      snapshot.delete:
+        repository: repository_temporary
+        snapshot: snapshot-one
+
+---
+"Register a repository with a non existing bucket":
+
+  - do:
+      catch: /repository_exception/
+      snapshot.create_repository:
+        repository: repository_temporary
+        body:
+          type: s3
+          settings:
+            bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
+            client: integration_test
+
+---
+"Register a repository with a non existing client":
+
+  - do:
+      catch: /repository_exception/
+      snapshot.create_repository:
+        repository: repository_temporary
+        body:
+          type: s3
+          settings:
+            bucket: repository_temporary
+            client: unknown
+
+---
+"Get a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.get:
+        repository: repository_temporary
+        snapshot: missing
+
+---
+"Delete a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.delete:
+        repository: repository_temporary
+        snapshot: missing
+
+---
+"Restore a non existing snapshot":
+
+  - do:
+      catch: /snapshot_restore_exception/
+      snapshot.restore:
+        repository: repository_temporary
+        snapshot: missing
+        wait_for_completion: true
+
+---
+teardown:
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: repository_temporary

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java

@@ -26,8 +26,10 @@ import java.util.Map;
 import java.util.Set;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.BasicAWSCredentials;
 
+import com.amazonaws.auth.BasicSessionCredentials;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.SecureString;
@@ -52,6 +54,10 @@ final class S3ClientSettings {
     static final Setting.AffixSetting<SecureString> SECRET_KEY_SETTING = Setting.affixKeySetting(PREFIX, "secret_key",
         key -> SecureSetting.secureString(key, null));
 
+    /** The secret key (ie password) for connecting to s3. */
+    static final Setting.AffixSetting<SecureString> SESSION_TOKEN_SETTING = Setting.affixKeySetting(PREFIX, "session_token",
+        key -> SecureSetting.secureString(key, null));
+
     /** An override for the s3 endpoint to connect to. */
     static final Setting.AffixSetting<String> ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint",
         key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope));
@@ -89,7 +95,7 @@ final class S3ClientSettings {
         key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope));
 
     /** Credentials to authenticate with s3. */
-    final BasicAWSCredentials credentials;
+    final AWSCredentials credentials;
 
     /** The s3 endpoint the client should talk to, or empty string to use the default. */
     final String endpoint;
@@ -120,7 +126,7 @@ final class S3ClientSettings {
     /** Whether the s3 client should use an exponential backoff retry policy. */
     final boolean throttleRetries;
 
-    protected S3ClientSettings(BasicAWSCredentials credentials, String endpoint, Protocol protocol,
+    protected S3ClientSettings(AWSCredentials credentials, String endpoint, Protocol protocol,
                              String proxyHost, int proxyPort, String proxyUsername, String proxyPassword,
                              int readTimeoutMillis, int maxRetries, boolean throttleRetries) {
         this.credentials = credentials;
@@ -190,26 +196,36 @@ final class S3ClientSettings {
         }
     }
 
-    static BasicAWSCredentials loadCredentials(Settings settings, String clientName) {
+    static AWSCredentials loadCredentials(Settings settings, String clientName) {
         try (SecureString accessKey = getConfigValue(settings, clientName, ACCESS_KEY_SETTING);
-                SecureString secretKey = getConfigValue(settings, clientName, SECRET_KEY_SETTING);) {
+             SecureString secretKey = getConfigValue(settings, clientName, SECRET_KEY_SETTING);
+             SecureString sessionToken = getConfigValue(settings, clientName, SESSION_TOKEN_SETTING)) {
             if (accessKey.length() != 0) {
                 if (secretKey.length() != 0) {
-                    return new BasicAWSCredentials(accessKey.toString(), secretKey.toString());
+                    if (sessionToken.length() != 0) {
+                        return new BasicSessionCredentials(accessKey.toString(), secretKey.toString(), sessionToken.toString());
+                    } else {
+                        return new BasicAWSCredentials(accessKey.toString(), secretKey.toString());
+                    }
                 } else {
                     throw new IllegalArgumentException("Missing secret key for s3 client [" + clientName + "]");
                 }
-            } else if (secretKey.length() != 0) {
-                throw new IllegalArgumentException("Missing access key for s3 client [" + clientName + "]");
+            } else {
+                if (secretKey.length() != 0) {
+                    throw new IllegalArgumentException("Missing access key for s3 client [" + clientName + "]");
+                }
+                if (sessionToken.length() != 0) {
+                    throw new IllegalArgumentException("Missing access key and secret key for s3 client [" + clientName + "]");
+                }
+                return null;
             }
-            return null;
         }
     }
 
     // pkg private for tests
     /** Parse settings for a single client. */
     static S3ClientSettings getClientSettings(Settings settings, String clientName) {
-        final BasicAWSCredentials credentials = S3ClientSettings.loadCredentials(settings, clientName);
+        final AWSCredentials credentials = S3ClientSettings.loadCredentials(settings, clientName);
         try (SecureString proxyUsername = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING);
              SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING)) {
             return new S3ClientSettings(

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java

@@ -92,6 +92,7 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo
             // named s3 client configuration settings
             S3ClientSettings.ACCESS_KEY_SETTING,
             S3ClientSettings.SECRET_KEY_SETTING,
+            S3ClientSettings.SESSION_TOKEN_SETTING,
             S3ClientSettings.ENDPOINT_SETTING,
             S3ClientSettings.PROTOCOL_SETTING,
             S3ClientSettings.PROXY_HOST_SETTING,

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -22,7 +22,6 @@ package org.elasticsearch.repositories.s3;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.InstanceProfileCredentialsProvider;
 import com.amazonaws.http.IdleConnectionReaper;
 import com.amazonaws.internal.StaticCredentialsProvider;
@@ -134,7 +133,7 @@ class S3Service extends AbstractComponent implements Closeable {
 
     // pkg private for tests
     static AWSCredentialsProvider buildCredentials(Logger logger, S3ClientSettings clientSettings) {
-        final BasicAWSCredentials credentials = clientSettings.credentials;
+        final AWSCredentials credentials = clientSettings.credentials;
         if (credentials == null) {
             logger.debug("Using instance profile credentials");
             return new PrivilegedInstanceProfileCredentialsProvider();

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java

@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.BasicSessionCredentials;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.isEmptyString;
+import static org.hamcrest.Matchers.nullValue;
+
+public class S3ClientSettingsTests extends ESTestCase {
+    public void testThereIsADefaultClientByDefault() {
+        final Map<String, S3ClientSettings> settings = S3ClientSettings.load(Settings.EMPTY);
+        assertThat(settings.keySet(), contains("default"));
+
+        final S3ClientSettings defaultSettings = settings.get("default");
+        assertThat(defaultSettings.credentials, nullValue());
+        assertThat(defaultSettings.endpoint, isEmptyString());
+        assertThat(defaultSettings.protocol, is(Protocol.HTTPS));
+        assertThat(defaultSettings.proxyHost, isEmptyString());
+        assertThat(defaultSettings.proxyPort, is(80));
+        assertThat(defaultSettings.proxyUsername, isEmptyString());
+        assertThat(defaultSettings.proxyPassword, isEmptyString());
+        assertThat(defaultSettings.readTimeoutMillis, is(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT));
+        assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry()));
+        assertThat(defaultSettings.throttleRetries, is(ClientConfiguration.DEFAULT_THROTTLE_RETRIES));
+    }
+
+    public void testDefaultClientSettingsCanBeSet() {
+        final Map<String, S3ClientSettings> settings = S3ClientSettings.load(Settings.builder()
+            .put("s3.client.default.max_retries", 10).build());
+        assertThat(settings.keySet(), contains("default"));
+
+        final S3ClientSettings defaultSettings = settings.get("default");
+        assertThat(defaultSettings.maxRetries, is(10));
+    }
+
+    public void testNondefaultClientCreatedBySettingItsSettings() {
+        final Map<String, S3ClientSettings> settings = S3ClientSettings.load(Settings.builder()
+            .put("s3.client.another_client.max_retries", 10).build());
+        assertThat(settings.keySet(), contains("default", "another_client"));
+
+        final S3ClientSettings defaultSettings = settings.get("default");
+        assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry()));
+
+        final S3ClientSettings anotherClientSettings = settings.get("another_client");
+        assertThat(anotherClientSettings.maxRetries, is(10));
+    }
+
+    public void testRejectionOfLoneAccessKey() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.access_key", "aws_key");
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()));
+        assertThat(e.getMessage(), is("Missing secret key for s3 client [default]"));
+    }
+
+    public void testRejectionOfLoneSecretKey() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.secret_key", "aws_key");
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()));
+        assertThat(e.getMessage(), is("Missing access key for s3 client [default]"));
+    }
+
+    public void testRejectionOfLoneSessionToken() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.session_token", "aws_key");
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()));
+        assertThat(e.getMessage(), is("Missing access key and secret key for s3 client [default]"));
+    }
+
+    public void testCredentialsTypeWithAccessKeyAndSecretKey() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.access_key", "access_key");
+        secureSettings.setString("s3.client.default.secret_key", "secret_key");
+        final Map<String, S3ClientSettings> settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build());
+        final S3ClientSettings defaultSettings = settings.get("default");
+        BasicAWSCredentials credentials = (BasicAWSCredentials) defaultSettings.credentials;
+        assertThat(credentials.getAWSAccessKeyId(), is("access_key"));
+        assertThat(credentials.getAWSSecretKey(), is("secret_key"));
+    }
+
+    public void testCredentialsTypeWithAccessKeyAndSecretKeyAndSessionToken() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.access_key", "access_key");
+        secureSettings.setString("s3.client.default.secret_key", "secret_key");
+        secureSettings.setString("s3.client.default.session_token", "session_token");
+        final Map<String, S3ClientSettings> settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build());
+        final S3ClientSettings defaultSettings = settings.get("default");
+        BasicSessionCredentials credentials = (BasicSessionCredentials) defaultSettings.credentials;
+        assertThat(credentials.getAWSAccessKeyId(), is("access_key"));
+        assertThat(credentials.getAWSSecretKey(), is("secret_key"));
+        assertThat(credentials.getSessionToken(), is("session_token"));
+    }
+}

server/src/main/java/org/elasticsearch/common/settings/SettingsException.java

@@ -42,4 +42,8 @@ public class SettingsException extends ElasticsearchException {
     public SettingsException(StreamInput in) throws IOException {
         super(in);
     }
+
+    public SettingsException(String msg, Object... args) {
+        super(msg, args);
+    }
 }