Use default application credentials for GCS repositories (#71239)

Adds support for "Default Application Credentials" for GCS repositories, making it easier to set up a repository on GCP,
as all relevant information to connect to the repository is retrieved from the environment, not necessitating complicated
keystore setups.

docs/plugins/repository-gcs.asciidoc

@@ -45,12 +45,16 @@ https://cloud.google.com/storage/docs/quickstart-console#create_a_bucket[Google
 
 The plugin must authenticate the requests it makes to the Google Cloud Storage
 service. It is common for Google client libraries to employ a strategy named https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application[application default credentials].
-However, that strategy is **not** supported for use with Elasticsearch. The
+However, that strategy is only **partially supported** by Elasticsearch. The
 plugin operates under the Elasticsearch process, which runs with the security
-manager enabled. The security manager obstructs the "automatic" credential discovery.
-Therefore, you must configure <<repository-gcs-using-service-account,service account>>
-credentials even if you are using an environment that does not normally require
-this configuration (such as Compute Engine, Kubernetes Engine or App Engine).
+manager enabled. The security manager obstructs the "automatic" credential discovery
+when the environment variable `GOOGLE_APPLICATION_CREDENTIALS` is used to point to a
+local file on disk. It can, however, retrieve the service account that is attached to
+the resource that is running Elasticsearch, or fall back to the default service
+account that Compute Engine, Kubernetes Engine or App Engine provide.
+Alternatively, you must configure <<repository-gcs-using-service-account,service account>>
+credentials if you are using an environment that does not support automatic
+credential discovery.
 
 [[repository-gcs-using-service-account]]
 ===== Using a Service Account

plugins/repository-gcs/build.gradle

@@ -220,6 +220,7 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
   apply plugin: 'elasticsearch.test.fixtures'
   testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture')
   testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party')
+  testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-with-application-default-credentials')
 
 } else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
   throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
@@ -266,28 +267,12 @@ tasks.named("internalClusterTest").configure {
   exclude '**/GoogleCloudStorageThirdPartyTests.class'
 }
 
-final Closure testClustersConfiguration = {
-  keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
-
-  if (useFixture) {
-    /* Use a closure on the string to delay evaluation until tests are executed */
-    setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE
-    setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE
-  } else {
-    println "Using an external service to test the repository-gcs plugin"
-  }
-}
-
 tasks.named("yamlRestTest").configure {
   if (useFixture) {
     dependsOn "createServiceAccountFile"
   }
 }
 
-testClusters {
-  all testClustersConfiguration
-}
-
 /*
  * We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add
  * an additional test that forces the large blob threshold to be small to exercise the resumable upload path.
@@ -330,6 +315,40 @@ def gcsThirdPartyTest = tasks.register("gcsThirdPartyTest", Test) {
   }
 }
 
+testClusters.matching {
+  it.name == "yamlRestTest" ||
+  it.name == "largeBlobYamlRestTest" ||
+  it.name == "gcsThirdPartyTest" }.configureEach {
+  keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
+
+  if (useFixture) {
+    /* Use a closure on the string to delay evaluation until tests are executed */
+    setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE
+    setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE
+  } else {
+    println "Using an external service to test the repository-gcs plugin"
+  }
+}
+
+
+// Application Default Credentials
+if (useFixture) {
+  tasks.register("yamlRestTestApplicationDefaultCredentials", RestIntegTestTask.class) {
+    dependsOn('bundlePlugin')
+    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+    SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
+    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
+    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
+  }
+  tasks.named("check").configure { dependsOn("yamlRestTestApplicationDefaultCredentials") }
+
+  testClusters.matching { it.name == "yamlRestTestApplicationDefaultCredentials" }.configureEach {
+    setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture-with-application-default-credentials')}" }, IGNORE_VALUE
+    plugin tasks.bundlePlugin.archiveFile
+    environment 'GCE_METADATA_HOST', { "${-> fixtureAddress('gcs-fixture-with-application-default-credentials')}".replace("http://", "") }, IGNORE_VALUE
+  }
+}
+
 tasks.named("check").configure {
   dependsOn(largeBlobYamlRestTest, gcsThirdPartyTest)
 }

plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java

@@ -12,6 +12,7 @@ import com.google.api.client.googleapis.GoogleUtils;
 import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.auth.oauth2.GoogleCredentials;
 import com.google.auth.oauth2.ServiceAccountCredentials;
 import com.google.cloud.ServiceOptions;
 import com.google.cloud.http.HttpTransportOptions;
@@ -21,14 +22,21 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.Maps;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
 import java.net.URI;
+import java.net.URL;
 import java.util.Map;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.Collections.emptyMap;
 
 public class GoogleCloudStorageService {
@@ -162,10 +170,37 @@ public class GoogleCloudStorageService {
         }
         if (Strings.hasLength(clientSettings.getProjectId())) {
             storageOptionsBuilder.setProjectId(clientSettings.getProjectId());
+        } else {
+            String defaultProjectId = null;
+            try {
+                defaultProjectId = ServiceOptions.getDefaultProjectId();
+                if (defaultProjectId != null) {
+                    storageOptionsBuilder.setProjectId(defaultProjectId);
+                }
+            } catch (Exception e) {
+                logger.warn("failed to load default project id", e);
+            }
+            if (defaultProjectId == null) {
+                try {
+                    // fallback to manually load project ID here as the above ServiceOptions method has the metadata endpoint hardcoded,
+                    // which makes it impossible to test
+                    SocketAccess.doPrivilegedVoidIOException(() -> {
+                        final String projectId = getDefaultProjectId();
+                        if (projectId != null) {
+                            storageOptionsBuilder.setProjectId(projectId);
+                        }
+                    });
+                } catch (Exception e) {
+                    logger.warn("failed to load default project id fallback", e);
+                }
+            }
         }
         if (clientSettings.getCredential() == null) {
-            logger.warn("\"Application Default Credentials\" are not supported out of the box."
-                    + " Additional file system permissions have to be granted to the plugin.");
+            try {
+                storageOptionsBuilder.setCredentials(GoogleCredentials.getApplicationDefault());
+            } catch (Exception e) {
+                logger.warn("failed to load Application Default Credentials", e);
+            }
         } else {
             ServiceAccountCredentials serviceAccountCredentials = clientSettings.getCredential();
             // override token server URI
@@ -180,6 +215,30 @@ public class GoogleCloudStorageService {
         return storageOptionsBuilder.build();
     }
 
+    /**
+     * This method imitates what MetadataConfig.getProjectId() does, but does not have the endpoint hardcoded.
+     */
+    @SuppressForbidden(reason = "ok to open connection here")
+    private static String getDefaultProjectId() throws IOException {
+        String metaHost = System.getenv("GCE_METADATA_HOST");
+        if (metaHost == null) {
+            metaHost = "metadata.google.internal";
+        }
+        URL url = new URL("http://" + metaHost + "/computeMetadata/v1/project/project-id");
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+        connection.setConnectTimeout(5000);
+        connection.setReadTimeout(5000);
+        connection.setRequestProperty("Metadata-Flavor", "Google");
+        try (InputStream input = connection.getInputStream()) {
+            if (connection.getResponseCode() == 200) {
+                try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, UTF_8))) {
+                    return reader.readLine();
+                }
+            }
+        }
+        return null;
+    }
+
     /**
      * Converts timeout values from the settings to a timeout value for the Google
      * Cloud SDK

test/fixtures/gcs-fixture/docker-compose.yml

@@ -60,3 +60,15 @@ services:
       - ./testfixtures_shared/shared:/fixture/shared
     ports:
       - "80"
+  gcs-fixture-with-application-default-credentials:
+    build:
+      context: .
+      args:
+        port: 80
+        bucket: "bucket"
+        token: "computeMetadata/v1/instance/service-accounts/default/token"
+      dockerfile: Dockerfile
+    volumes:
+      - ./testfixtures_shared/shared:/fixture/shared
+    ports:
+      - "80"

test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java

@@ -27,6 +27,7 @@ public class FakeOAuth2HttpHandler implements HttpHandler {
             while (exchange.getRequestBody().read(BUFFER) >= 0) ;
             byte[] response = ("{\"access_token\":\"foo\",\"token_type\":\"Bearer\",\"expires_in\":3600}").getBytes(UTF_8);
             exchange.getResponseHeaders().add("Content-Type", "application/json");
+            exchange.getResponseHeaders().add("Metadata-Flavor", "Google");
             exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
             exchange.getResponseBody().write(response);
         } finally {

test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeProjectIdHttpHandler.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package fixture.gcs;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+@SuppressForbidden(reason = "Uses a HttpServer to emulate a fake metadata service")
+public class FakeProjectIdHttpHandler implements HttpHandler {
+
+    private static final byte[] BUFFER = new byte[1024];
+
+    @Override
+    public void handle(final HttpExchange exchange) throws IOException {
+        try {
+            while (exchange.getRequestBody().read(BUFFER) >= 0) ;
+            byte[] response = ("some-project-id").getBytes(UTF_8);
+            exchange.getResponseHeaders().add("Content-Type", "application/json");
+            exchange.getResponseHeaders().add("Metadata-Flavor", "Google");
+            exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
+            exchange.getResponseBody().write(response);
+        } finally {
+            int read = exchange.getRequestBody().read();
+            assert read == -1 : "Request body should have been fully read here but saw [" + read + "]";
+            exchange.close();
+        }
+    }
+}

test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpFixture.java

@@ -21,6 +21,7 @@ public class GoogleCloudStorageHttpFixture {
                                           final String bucket, final String token) throws IOException {
         this.server = HttpServer.create(new InetSocketAddress(InetAddress.getByName(address), port), 0);
         server.createContext("/" + token, new FakeOAuth2HttpHandler());
+        server.createContext("/computeMetadata/v1/project/project-id", new FakeProjectIdHttpHandler());
         server.createContext("/", new GoogleCloudStorageHttpHandler(bucket));
     }
 

test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java

@@ -76,7 +76,12 @@ public class GoogleCloudStorageHttpHandler implements HttpHandler {
         try {
             // Request body is closed in the finally block
             final BytesReference requestBody = Streams.readFully(Streams.noCloseStream(exchange.getRequestBody()));
-            if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "/o/*", request)) {
+            if (request.equals("GET /") &&
+                "Google".equals(exchange.getRequestHeaders().getFirst("Metadata-Flavor"))) {
+                // the SDK checks this endpoint to determine if it's running within Google Compute Engine
+                exchange.getResponseHeaders().add("Metadata-Flavor", "Google");
+                exchange.sendResponseHeaders(RestStatus.OK.getStatus(), 0);
+            } else if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "/o/*", request)) {
                 final String key = exchange.getRequestURI().getPath().replace("/storage/v1/b/" + bucket + "/o/", "");
                 final BytesReference blob = blobs.get(key);
                 if (blob == null) {

test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java

@@ -244,8 +244,12 @@ public abstract class ESMockAPIBasedRepositoryIntegTestCase extends ESBlobStoreR
     }
 
     protected static String httpServerUrl() {
+        return "http://" + serverUrl();
+    }
+
+    protected static String serverUrl() {
         InetSocketAddress address = httpServer.getAddress();
-        return "http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort();
+        return InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort();
     }
 
     /**