Discovery EC2: Utilize Amazon SDK to resolve EC2 metadata server (#35246)

* Discovery EC2: Utilize Amazon SDK to resolve EC2 metadata server

Closes #35141

plugins/discovery-ec2/build.gradle

@@ -48,9 +48,30 @@ bundlePlugin {
   }
 }
 
+task writeTestJavaPolicy {
+  doLast {
+    final File tmp = file("${buildDir}/tmp")
+    if (tmp.exists() == false && tmp.mkdirs() == false) {
+      throw new GradleException("failed to create temporary directory [${tmp}]")
+    }
+    final File javaPolicy = file("${tmp}/java.policy")
+    javaPolicy.write(
+    [
+            "grant {",
+            "  permission java.util.PropertyPermission \"com.amazonaws.sdk.ec2MetadataServiceEndpointOverride\", \"write\";",
+            "};"
+    ].join("\n"))
+  }
+}
+
 test {
+  dependsOn writeTestJavaPolicy
   // this is needed for insecure plugins, remove if possible!
   systemProperty 'tests.artifact', project.name
+  
+  // this is needed to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property
+  // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'`
+  systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy"
 }
 
 check {

plugins/discovery-ec2/qa/amazon-ec2/build.gradle

@@ -55,7 +55,10 @@ integTestCluster {
     keystoreSetting 'discovery.ec2.access_key', 'ec2_integration_test_access_key'
     keystoreSetting 'discovery.ec2.secret_key', 'ec2_integration_test_secret_key'
     setting 'discovery.zen.hosts_provider', 'ec2'
+    setting 'network.host', '_ec2_'
     setting 'discovery.ec2.endpoint', "http://${-> ec2Fixture.addressAndPort}"
+    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", "http://${-> ec2Fixture.addressAndPort}"
+    
     unicastTransportUri = { seedNode, node, ant -> return null }
 
     waitCondition = { node, ant ->

plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java

@@ -19,6 +19,8 @@
 package org.elasticsearch.discovery.ec2;
 
 import org.apache.http.NameValuePair;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.utils.URLEncodedUtils;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.rest.RestStatus;
@@ -60,7 +62,7 @@ public class AmazonEC2Fixture extends AbstractHttpFixture {
 
     @Override
     protected Response handle(final Request request) throws IOException {
-        if ("/".equals(request.getPath()) && ("POST".equals(request.getMethod()))) {
+        if ("/".equals(request.getPath()) && (HttpPost.METHOD_NAME.equals(request.getMethod()))) {
             final String userAgent = request.getHeader("User-Agent");
             if (userAgent != null && userAgent.startsWith("aws-sdk-java")) {
                 // Simulate an EC2 DescribeInstancesResponse
@@ -74,6 +76,9 @@ public class AmazonEC2Fixture extends AbstractHttpFixture {
                 return new Response(RestStatus.OK.getStatus(), contentType("text/xml; charset=UTF-8"), responseBody);
             }
         }
+        if ("/latest/meta-data/local-ipv4".equals(request.getPath()) && (HttpGet.METHOD_NAME.equals(request.getMethod()))) {
+            return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, "127.0.0.1".getBytes(UTF_8));
+        }
         return null;
     }
 

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java

@@ -40,8 +40,6 @@ import java.util.concurrent.atomic.AtomicReference;
 
 class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service {
 
-    public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/";
-
     private final AtomicReference<LazyInitializable<AmazonEc2Reference, ElasticsearchException>> lazyClientReference =
             new AtomicReference<>();
 

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import com.amazonaws.util.EC2MetadataUtils;
 import com.amazonaws.util.json.Jackson;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -129,7 +130,8 @@ public class Ec2DiscoveryPlugin extends Plugin implements DiscoveryPlugin, Reloa
         final Settings.Builder builder = Settings.builder();
 
         // Adds a node attribute for the ec2 availability zone
-        final String azMetadataUrl = AwsEc2ServiceImpl.EC2_METADATA_URL + "placement/availability-zone";
+        final String azMetadataUrl = EC2MetadataUtils.getHostAddressForEC2MetadataService()
+            + "/latest/meta-data/placement/availability-zone";
         builder.put(getAvailabilityZoneNodeAttributes(settings, azMetadataUrl));
         return builder.build();
     }

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import com.amazonaws.util.EC2MetadataUtils;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -86,7 +87,7 @@ class Ec2NameResolver extends AbstractComponent implements CustomNameResolver {
     @SuppressForbidden(reason = "We call getInputStream in doPrivileged and provide SocketPermission")
     public InetAddress[] resolve(Ec2HostnameType type) throws IOException {
         InputStream in = null;
-        String metadataUrl = AwsEc2ServiceImpl.EC2_METADATA_URL + type.ec2Name;
+        String metadataUrl = EC2MetadataUtils.getHostAddressForEC2MetadataService() + "/latest/meta-data/" + type.ec2Name;
         try {
             URL url = new URL(metadataUrl);
             logger.debug("obtaining ec2 hostname from ec2 meta-data url {}", url);

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java

@@ -19,16 +19,32 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import com.sun.net.httpserver.HttpServer;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.mocksocket.MockHttpServer;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.function.BiConsumer;
 
+import static com.amazonaws.SDKGlobalConfiguration.EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 /**
  * Test for EC2 network.host settings.
@@ -36,22 +52,65 @@ import static org.hamcrest.Matchers.containsString;
  * Warning: This test doesn't assert that the exceptions are thrown.
  * They aren't.
  */
+@SuppressForbidden(reason = "use http server")
 public class Ec2NetworkTests extends ESTestCase {
+
+    private static HttpServer httpServer;
+
+    @BeforeClass
+    public static void startHttp() throws Exception {
+        httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
+
+        BiConsumer<String, String> registerContext = (path, v) ->{
+            final byte[] message = v.getBytes(UTF_8);
+            httpServer.createContext(path, (s) -> {
+                s.sendResponseHeaders(RestStatus.OK.getStatus(), message.length);
+                OutputStream responseBody = s.getResponseBody();
+                responseBody.write(message);
+                responseBody.close();
+            });
+        };
+        registerContext.accept("/latest/meta-data/local-ipv4","127.0.0.1");
+        registerContext.accept("/latest/meta-data/public-ipv4","165.168.10.2");
+        registerContext.accept("/latest/meta-data/public-hostname","165.168.10.3");
+        registerContext.accept("/latest/meta-data/local-hostname","10.10.10.5");
+
+        httpServer.start();
+    }
+
+    @Before
+    public void setup() {
+        // redirect EC2 metadata service to httpServer
+        AccessController.doPrivileged((PrivilegedAction<String>) () -> System.setProperty(EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY,
+            "http://" + httpServer.getAddress().getHostName() + ":" + httpServer.getAddress().getPort()));
+    }
+
+    @AfterClass
+    public static void stopHttp() {
+        httpServer.stop(0);
+        httpServer = null;
+    }
+
     /**
      * Test for network.host: _ec2_
      */
     public void testNetworkHostEc2() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2_")
-                .build();
+        resolveEc2("_ec2_", InetAddress.getByName("127.0.0.1"));
+    }
+
+    /**
+     * Test for network.host: _ec2_
+     */
+    public void testNetworkHostUnableToResolveEc2() {
+        // redirect EC2 metadata service to unknown location
+        AccessController.doPrivileged((PrivilegedAction<String>) () -> System.setProperty(EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY,
+            "http://127.0.0.1/"));
 
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
         try {
-            networkService.resolveBindHostAddresses(null);
-            // note: this can succeed and the test can pass
+            resolveEc2("_ec2_", (InetAddress[]) null);
         } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("local-ipv4"));
+            assertThat(e.getMessage(),
+                equalTo("IOException caught when fetching InetAddress from [http://127.0.0.1//latest/meta-data/local-ipv4]"));
         }
     }
 
@@ -59,108 +118,58 @@ public class Ec2NetworkTests extends ESTestCase {
      * Test for network.host: _ec2:publicIp_
      */
     public void testNetworkHostEc2PublicIp() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:publicIp_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-            // note: this can succeed and the test can pass
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("public-ipv4"));
-        }
+        resolveEc2("_ec2:publicIp_", InetAddress.getByName("165.168.10.2"));
     }
 
     /**
      * Test for network.host: _ec2:privateIp_
      */
     public void testNetworkHostEc2PrivateIp() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:privateIp_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-            // note: this can succeed and the test can pass
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("local-ipv4"));
-        }
+        resolveEc2("_ec2:privateIp_", InetAddress.getByName("127.0.0.1"));
     }
 
     /**
      * Test for network.host: _ec2:privateIpv4_
      */
     public void testNetworkHostEc2PrivateIpv4() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:privateIpv4_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-            // note: this can succeed and the test can pass
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("local-ipv4"));
-        }
+        resolveEc2("_ec2:privateIpv4_", InetAddress.getByName("127.0.0.1"));
     }
 
     /**
      * Test for network.host: _ec2:privateDns_
      */
     public void testNetworkHostEc2PrivateDns() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:privateDns_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-            // note: this can succeed and the test can pass
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("local-hostname"));
-        }
+        resolveEc2("_ec2:privateDns_", InetAddress.getByName("10.10.10.5"));
     }
 
     /**
      * Test for network.host: _ec2:publicIpv4_
      */
     public void testNetworkHostEc2PublicIpv4() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:publicIpv4_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-            // note: this can succeed and the test can pass
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("public-ipv4"));
-        }
+        resolveEc2("_ec2:publicIpv4_", InetAddress.getByName("165.168.10.2"));
     }
 
     /**
      * Test for network.host: _ec2:publicDns_
      */
     public void testNetworkHostEc2PublicDns() throws IOException {
+        resolveEc2("_ec2:publicDns_", InetAddress.getByName("165.168.10.3"));
+    }
+
+    private InetAddress[] resolveEc2(String host, InetAddress ... expected) throws IOException {
         Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:publicDns_")
-                .build();
+            .put("network.host", host)
+            .build();
 
         NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-            // note: this can succeed and the test can pass
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("public-hostname"));
+
+        InetAddress[] addresses = networkService.resolveBindHostAddresses(
+            NetworkService.GLOBAL_NETWORK_BINDHOST_SETTING.get(nodeSettings).toArray(Strings.EMPTY_ARRAY));
+        if (expected == null) {
+            fail("We should get an IOException, resolved addressed:" + Arrays.toString(addresses));
         }
+        assertThat(addresses, arrayContaining(expected));
+        return addresses;
     }
 
     /**