Log unsuccessful attempts to get credentials from web identity tokens (#88241)

Currently, we only verify that local environment for web identity tokens is correctly set up, but we don't verify whether it's
possible to exchange the token to credentials from the STS. If we can't get credentials from the STS, we silently fall back
to the EC2 credentials provider. Let's try to log the web identity token auth errors, so the users get a clear message in the logs in case the STS is unavailable for the ES server.

docs/changelog/88241.yaml

@@ -0,0 +1,5 @@
+pr: 88241
+summary: Log unsuccessful attempts to get credentials from web identity tokens
+area: Allocation
+type: enhancement
+issues: []

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -39,6 +39,7 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Clock;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -223,7 +224,12 @@ class S3Service implements Closeable {
             if (webIdentityTokenCredentialsProvider.isActive()) {
                 logger.debug("Using a custom provider chain of Web Identity Token and instance profile credentials");
                 return new PrivilegedAWSCredentialsProvider(
-                    new AWSCredentialsProviderChain(webIdentityTokenCredentialsProvider, new EC2ContainerCredentialsProviderWrapper())
+                    new AWSCredentialsProviderChain(
+                        List.of(
+                            new ErrorLoggingCredentialsProvider(webIdentityTokenCredentialsProvider, LOGGER),
+                            new ErrorLoggingCredentialsProvider(new EC2ContainerCredentialsProviderWrapper(), LOGGER)
+                        )
+                    )
                 );
             } else {
                 logger.debug("Using instance profile credentials");
@@ -375,6 +381,37 @@ class S3Service implements Closeable {
         }
     }
 
+    static class ErrorLoggingCredentialsProvider implements AWSCredentialsProvider {
+
+        private final AWSCredentialsProvider delegate;
+        private final Logger logger;
+
+        ErrorLoggingCredentialsProvider(AWSCredentialsProvider delegate, Logger logger) {
+            this.delegate = Objects.requireNonNull(delegate);
+            this.logger = Objects.requireNonNull(logger);
+        }
+
+        @Override
+        public AWSCredentials getCredentials() {
+            try {
+                return delegate.getCredentials();
+            } catch (Exception e) {
+                logger.error(() -> "Unable to load credentials from " + delegate, e);
+                throw e;
+            }
+        }
+
+        @Override
+        public void refresh() {
+            try {
+                delegate.refresh();
+            } catch (Exception e) {
+                logger.error(() -> "Unable to refresh " + delegate, e);
+                throw e;
+            }
+        }
+    }
+
     @FunctionalInterface
     interface SystemEnvironment {
         String getEnv(String name);

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java

@@ -17,17 +17,22 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
 
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
 
 public class AwsS3ServiceImplTests extends ESTestCase {
 
@@ -229,4 +234,40 @@ public class AwsS3ServiceImplTests extends ESTestCase {
         assertThat(clientSettings.endpoint, is(expectedEndpoint));
     }
 
+    public void testLoggingCredentialsProviderCatchesErrors() {
+        var mockProvider = Mockito.mock(AWSCredentialsProvider.class);
+        String mockProviderErrorMessage = "mockProvider failed to generate credentials";
+        Mockito.when(mockProvider.getCredentials()).thenThrow(new IllegalStateException(mockProviderErrorMessage));
+        var mockLogger = Mockito.mock(Logger.class);
+
+        var credentialsProvider = new S3Service.ErrorLoggingCredentialsProvider(mockProvider, mockLogger);
+        var exception = expectThrows(IllegalStateException.class, credentialsProvider::getCredentials);
+        assertEquals(mockProviderErrorMessage, exception.getMessage());
+
+        var messageSupplierCaptor = ArgumentCaptor.forClass(Supplier.class);
+        var throwableCaptor = ArgumentCaptor.forClass(Throwable.class);
+        Mockito.verify(mockLogger).error(messageSupplierCaptor.capture(), throwableCaptor.capture());
+
+        assertThat(messageSupplierCaptor.getValue().get().toString(), startsWith("Unable to load credentials from"));
+        assertThat(throwableCaptor.getValue().getMessage(), equalTo(mockProviderErrorMessage));
+    }
+
+    public void testLoggingCredentialsProviderCatchesErrorsOnRefresh() {
+        var mockProvider = Mockito.mock(AWSCredentialsProvider.class);
+        String mockProviderErrorMessage = "mockProvider failed to refresh";
+        Mockito.doThrow(new IllegalStateException(mockProviderErrorMessage)).when(mockProvider).refresh();
+        var mockLogger = Mockito.mock(Logger.class);
+
+        var credentialsProvider = new S3Service.ErrorLoggingCredentialsProvider(mockProvider, mockLogger);
+        var exception = expectThrows(IllegalStateException.class, credentialsProvider::refresh);
+        assertEquals(mockProviderErrorMessage, exception.getMessage());
+
+        var messageSupplierCaptor = ArgumentCaptor.forClass(Supplier.class);
+        var throwableCaptor = ArgumentCaptor.forClass(Throwable.class);
+        Mockito.verify(mockLogger).error(messageSupplierCaptor.capture(), throwableCaptor.capture());
+
+        assertThat(messageSupplierCaptor.getValue().get().toString(), startsWith("Unable to refresh"));
+        assertThat(throwableCaptor.getValue().getMessage(), equalTo(mockProviderErrorMessage));
+    }
+
 }