Allow binding to multiple addresses

* Allow for multiple host specifications (e.g. _en0_,192.168.1.2,_site_).
* Add _site_ and _global_ scopes as counterparts to _local_.
* Warn on heuristic selection of publish address.
* Remove the arbitrary _non_loopback_ setting.

Closes #13954

core/src/main/java/org/elasticsearch/common/network/NetworkService.java

@@ -27,6 +27,9 @@ import org.elasticsearch.common.unit.TimeValue;
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
@@ -90,13 +93,21 @@ public class NetworkService extends AbstractComponent {
         customNameResolvers.add(customNameResolver);
     }
 
-    public InetAddress[] resolveBindHostAddress(String bindHost) throws IOException {
+    /**
+     * Resolves {@code bindHosts} to a list of internet addresses. The list will
+     * not contain duplicate addresses.
+     * @param bindHosts list of hosts to bind to. this may contain special pseudo-hostnames
+     *                  such as _local_ (see the documentation). if it is null, it will be populated
+     *                  based on global default settings.
+     * @return unique set of internet addresses
+     */
+    public InetAddress[] resolveBindHostAddresses(String bindHosts[]) throws IOException {
         // first check settings
-        if (bindHost == null) {
-            bindHost = settings.get(GLOBAL_NETWORK_BINDHOST_SETTING, settings.get(GLOBAL_NETWORK_HOST_SETTING));
+        if (bindHosts == null) {
+            bindHosts = settings.getAsArray(GLOBAL_NETWORK_BINDHOST_SETTING, settings.getAsArray(GLOBAL_NETWORK_HOST_SETTING, null));
         }
         // next check any registered custom resolvers
-        if (bindHost == null) {
+        if (bindHosts == null) {
             for (CustomNameResolver customNameResolver : customNameResolvers) {
                 InetAddress addresses[] = customNameResolver.resolveDefault();
                 if (addresses != null) {
@@ -105,31 +116,44 @@ public class NetworkService extends AbstractComponent {
             }
         }
         // finally, fill with our default
-        if (bindHost == null) {
-            bindHost = DEFAULT_NETWORK_HOST;
+        if (bindHosts == null) {
+            bindHosts = new String[] { DEFAULT_NETWORK_HOST };
         }
-        InetAddress addresses[] = resolveInetAddress(bindHost);
+        InetAddress addresses[] = resolveInetAddresses(bindHosts);
 
         // try to deal with some (mis)configuration
-        if (addresses != null) {
-            for (InetAddress address : addresses) {
-                // check if its multicast: flat out mistake
-                if (address.isMulticastAddress()) {
-                    throw new IllegalArgumentException("bind address: {" + NetworkAddress.format(address) + "} is invalid: multicast address");
-                }
+        for (InetAddress address : addresses) {
+            // check if its multicast: flat out mistake
+            if (address.isMulticastAddress()) {
+                throw new IllegalArgumentException("bind address: {" + NetworkAddress.format(address) + "} is invalid: multicast address");
+            }
+            // check if its a wildcard address: this is only ok if its the only address!
+            if (address.isAnyLocalAddress() && addresses.length > 1) {
+                throw new IllegalArgumentException("bind address: {" + NetworkAddress.format(address) + "} is wildcard, but multiple addresses specified: this makes no sense");
             }
         }
         return addresses;
     }
 
+    /**
+     * Resolves {@code publishHosts} to a single publish address. The fact that it returns
+     * only one address is just a current limitation.
+     * <p>
+     * If {@code publishHosts} resolves to more than one address, <b>then one is selected with magic</b>,
+     * and the user is warned (they can always just be more specific).
+     * @param publishHosts list of hosts to publish as. this may contain special pseudo-hostnames
+     *                     such as _local_ (see the documentation). if it is null, it will be populated
+     *                     based on global default settings.
+     * @return single internet address
+     */
     // TODO: needs to be InetAddress[]
-    public InetAddress resolvePublishHostAddress(String publishHost) throws IOException {
+    public InetAddress resolvePublishHostAddresses(String publishHosts[]) throws IOException {
         // first check settings
-        if (publishHost == null) {
-            publishHost = settings.get(GLOBAL_NETWORK_PUBLISHHOST_SETTING, settings.get(GLOBAL_NETWORK_HOST_SETTING));
+        if (publishHosts == null) {
+            publishHosts = settings.getAsArray(GLOBAL_NETWORK_PUBLISHHOST_SETTING, settings.getAsArray(GLOBAL_NETWORK_HOST_SETTING, null));
         }
         // next check any registered custom resolvers
-        if (publishHost == null) {
+        if (publishHosts == null) {
             for (CustomNameResolver customNameResolver : customNameResolvers) {
                 InetAddress addresses[] = customNameResolver.resolveDefault();
                 if (addresses != null) {
@@ -138,30 +162,59 @@ public class NetworkService extends AbstractComponent {
             }
         }
         // finally, fill with our default
-        if (publishHost == null) {
-            publishHost = DEFAULT_NETWORK_HOST;
+        if (publishHosts == null) {
+            publishHosts = new String[] { DEFAULT_NETWORK_HOST };
         }
+        InetAddress addresses[] = resolveInetAddresses(publishHosts);
         // TODO: allow publishing multiple addresses
-        InetAddress address = resolveInetAddress(publishHost)[0];
+        // for now... the hack begins
 
-        // try to deal with some (mis)configuration
-        if (address != null) {
+        // 1. single wildcard address, probably set by network.host: expand to all interface addresses.
+        if (addresses.length == 1 && addresses[0].isAnyLocalAddress()) {
+            HashSet<InetAddress> all = new HashSet<>(Arrays.asList(NetworkUtils.getAllAddresses()));
+            addresses = all.toArray(new InetAddress[all.size()]);
+        }
+
+        // 2. try to deal with some (mis)configuration
+        for (InetAddress address : addresses) {
             // check if its multicast: flat out mistake
             if (address.isMulticastAddress()) {
                 throw new IllegalArgumentException("publish address: {" + NetworkAddress.format(address) + "} is invalid: multicast address");
             }
-            // wildcard address, probably set by network.host
+            // check if its a wildcard address: this is only ok if its the only address!
+            // (if it was a single wildcard address, it was replaced by step 1 above)
             if (address.isAnyLocalAddress()) {
-                InetAddress old = address;
-                address = NetworkUtils.getFirstNonLoopbackAddresses()[0];
-                logger.warn("publish address: {{}} is a wildcard address, falling back to first non-loopback: {{}}", 
-                            NetworkAddress.format(old), NetworkAddress.format(address));
+                throw new IllegalArgumentException("publish address: {" + NetworkAddress.format(address) + "} is wildcard, but multiple addresses specified: this makes no sense");
             }
         }
-        return address;
+        
+        // 3. warn user if we end out with multiple publish addresses
+        if (addresses.length > 1) {
+            List<InetAddress> sorted = new ArrayList<>(Arrays.asList(addresses));
+            NetworkUtils.sortAddresses(sorted);
+            addresses = new InetAddress[] { sorted.get(0) };
+            logger.warn("publish host: {} resolves to multiple addresses, auto-selecting {{}} as single publish address", 
+                    Arrays.toString(publishHosts), NetworkAddress.format(addresses[0]));
+        }
+        return addresses[0];
+    }
+    
+    /** resolves (and deduplicates) host specification */
+    private InetAddress[] resolveInetAddresses(String hosts[]) throws IOException {
+        if (hosts.length == 0) {
+            throw new IllegalArgumentException("empty host specification");
+        }
+        // deduplicate, in case of resolver misconfiguration
+        // stuff like https://bugzilla.redhat.com/show_bug.cgi?id=496300
+        HashSet<InetAddress> set = new HashSet<>();
+        for (String host : hosts) {
+            set.addAll(Arrays.asList(resolveInternal(host)));
+        }
+        return set.toArray(new InetAddress[set.size()]);
     }
 
-    private InetAddress[] resolveInetAddress(String host) throws IOException {
+    /** resolves a single host specification */
+    private InetAddress[] resolveInternal(String host) throws IOException {
         if ((host.startsWith("#") && host.endsWith("#")) || (host.startsWith("_") && host.endsWith("_"))) {
             host = host.substring(1, host.length() - 1);
             // allow custom resolvers to have special names
@@ -178,12 +231,18 @@ public class NetworkService extends AbstractComponent {
                     return NetworkUtils.filterIPV4(NetworkUtils.getLoopbackAddresses());
                 case "local:ipv6":
                     return NetworkUtils.filterIPV6(NetworkUtils.getLoopbackAddresses());
-                case "non_loopback":
-                    return NetworkUtils.getFirstNonLoopbackAddresses();
-                case "non_loopback:ipv4":
-                    return NetworkUtils.filterIPV4(NetworkUtils.getFirstNonLoopbackAddresses());
-                case "non_loopback:ipv6":
-                    return NetworkUtils.filterIPV6(NetworkUtils.getFirstNonLoopbackAddresses());
+                case "site":
+                    return NetworkUtils.getSiteLocalAddresses();
+                case "site:ipv4":
+                    return NetworkUtils.filterIPV4(NetworkUtils.getSiteLocalAddresses());
+                case "site:ipv6":
+                    return NetworkUtils.filterIPV6(NetworkUtils.getSiteLocalAddresses());
+                case "global":
+                    return NetworkUtils.getGlobalAddresses();
+                case "global:ipv4":
+                    return NetworkUtils.filterIPV4(NetworkUtils.getGlobalAddresses());
+                case "global:ipv6":
+                    return NetworkUtils.filterIPV6(NetworkUtils.getGlobalAddresses());
                 default:
                     /* an interface specification */
                     if (host.endsWith(":ipv4")) {
@@ -197,6 +256,6 @@ public class NetworkService extends AbstractComponent {
                     }
             }
         }
-        return NetworkUtils.getAllByName(host);
+        return InetAddress.getAllByName(host);
     }
 }

core/src/main/java/org/elasticsearch/common/network/NetworkUtils.java

@@ -27,12 +27,10 @@ import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.NetworkInterface;
 import java.net.SocketException;
-import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.HashSet;
 import java.util.List;
 
 /**
@@ -109,7 +107,8 @@ public abstract class NetworkUtils {
      * @deprecated remove this when multihoming is really correct
      */
     @Deprecated
-    static void sortAddresses(List<InetAddress> list) {
+    // only public because of silly multicast
+    public static void sortAddresses(List<InetAddress> list) {
         Collections.sort(list, new Comparator<InetAddress>() {
             @Override
             public int compare(InetAddress left, InetAddress right) {
@@ -150,34 +149,79 @@ public abstract class NetworkUtils {
         return Constants.WINDOWS ? false : true;
     }
     
-    /** Returns addresses for all loopback interfaces that are up. */
+    /** Returns all interface-local scope (loopback) addresses for interfaces that are up. */
     static InetAddress[] getLoopbackAddresses() throws SocketException {
         List<InetAddress> list = new ArrayList<>();
         for (NetworkInterface intf : getInterfaces()) {
-            if (intf.isLoopback() && intf.isUp()) {
-                list.addAll(Collections.list(intf.getInetAddresses()));
+            if (intf.isUp()) {
+                // NOTE: some operating systems (e.g. BSD stack) assign a link local address to the loopback interface
+                // while technically not a loopback address, some of these treat them as one (e.g. OS X "localhost") so we must too,
+                // otherwise things just won't work out of box. So we include all addresses from loopback interfaces.
+                for (InetAddress address : Collections.list(intf.getInetAddresses())) {
+                    if (intf.isLoopback() || address.isLoopbackAddress()) {
+                        list.add(address);
+                    }
+                }
             }
         }
         if (list.isEmpty()) {
-            throw new IllegalArgumentException("No up-and-running loopback interfaces found, got " + getInterfaces());
+            throw new IllegalArgumentException("No up-and-running loopback addresses found, got " + getInterfaces());
         }
-        sortAddresses(list);
         return list.toArray(new InetAddress[list.size()]);
     }
     
-    /** Returns addresses for the first non-loopback interface that is up. */
-    static InetAddress[] getFirstNonLoopbackAddresses() throws SocketException {
+    /** Returns all site-local scope (private) addresses for interfaces that are up. */
+    static InetAddress[] getSiteLocalAddresses() throws SocketException {
         List<InetAddress> list = new ArrayList<>();
         for (NetworkInterface intf : getInterfaces()) {
-            if (intf.isLoopback() == false && intf.isUp()) {
-                list.addAll(Collections.list(intf.getInetAddresses()));
-                break;
+            if (intf.isUp()) {
+                for (InetAddress address : Collections.list(intf.getInetAddresses())) {
+                    if (address.isSiteLocalAddress()) {
+                        list.add(address);
+                    }
+                }
             }
         }
         if (list.isEmpty()) {
-            throw new IllegalArgumentException("No up-and-running non-loopback interfaces found, got " + getInterfaces());
+            throw new IllegalArgumentException("No up-and-running site-local (private) addresses found, got " + getInterfaces());
+        }
+        return list.toArray(new InetAddress[list.size()]);
+    }
+    
+    /** Returns all global scope addresses for interfaces that are up. */
+    static InetAddress[] getGlobalAddresses() throws SocketException {
+        List<InetAddress> list = new ArrayList<>();
+        for (NetworkInterface intf : getInterfaces()) {
+            if (intf.isUp()) {
+                for (InetAddress address : Collections.list(intf.getInetAddresses())) {
+                    if (address.isLoopbackAddress() == false && 
+                        address.isSiteLocalAddress() == false &&
+                        address.isLinkLocalAddress() == false) {
+                        list.add(address);
+                    }
+                }
+            }
+        }
+        if (list.isEmpty()) {
+            throw new IllegalArgumentException("No up-and-running global-scope (public) addresses found, got " + getInterfaces());
+        }
+        return list.toArray(new InetAddress[list.size()]);
+    }
+    
+    /** Returns all addresses (any scope) for interfaces that are up. 
+     *  This is only used to pick a publish address, when the user set network.host to a wildcard */
+    static InetAddress[] getAllAddresses() throws SocketException {
+        List<InetAddress> list = new ArrayList<>();
+        for (NetworkInterface intf : getInterfaces()) {
+            if (intf.isUp()) {
+                for (InetAddress address : Collections.list(intf.getInetAddresses())) {
+                    list.add(address);
+                }
+            }
+        }
+        if (list.isEmpty()) {
+            throw new IllegalArgumentException("No up-and-running addresses found, got " + getInterfaces());
         }
-        sortAddresses(list);
         return list.toArray(new InetAddress[list.size()]);
     }
     
@@ -194,20 +238,9 @@ public abstract class NetworkUtils {
         if (list.isEmpty()) {
             throw new IllegalArgumentException("Interface '" + name + "' has no internet addresses");
         }
-        sortAddresses(list);
         return list.toArray(new InetAddress[list.size()]);
     }
     
-    /** Returns addresses for the given host, sorted by order of preference */
-    static InetAddress[] getAllByName(String host) throws UnknownHostException {
-        InetAddress addresses[] = InetAddress.getAllByName(host);
-        // deduplicate, in case of resolver misconfiguration
-        // stuff like https://bugzilla.redhat.com/show_bug.cgi?id=496300
-        List<InetAddress> unique = new ArrayList<>(new HashSet<>(Arrays.asList(addresses)));
-        sortAddresses(unique);
-        return unique.toArray(new InetAddress[unique.size()]);
-    }
-    
     /** Returns only the IPV4 addresses in {@code addresses} */
     static InetAddress[] filterIPV4(InetAddress addresses[]) {
         List<InetAddress> list = new ArrayList<>();

core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java

@@ -50,6 +50,7 @@ import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicReference;
@@ -105,9 +106,9 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
 
     protected final String port;
 
-    protected final String bindHost;
+    protected final String bindHosts[];
 
-    protected final String publishHost;
+    protected final String publishHosts[];
 
     protected final boolean detailedErrorsEnabled;
 
@@ -157,8 +158,8 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
         this.workerCount = settings.getAsInt("http.netty.worker_count", EsExecutors.boundedNumberOfProcessors(settings) * 2);
         this.blockingServer = settings.getAsBoolean("http.netty.http.blocking_server", settings.getAsBoolean(TCP_BLOCKING_SERVER, settings.getAsBoolean(TCP_BLOCKING, false)));
         this.port = settings.get("http.netty.port", settings.get("http.port", "9200-9300"));
-        this.bindHost = settings.get("http.netty.bind_host", settings.get("http.bind_host", settings.get("http.host")));
-        this.publishHost = settings.get("http.netty.publish_host", settings.get("http.publish_host", settings.get("http.host")));
+        this.bindHosts = settings.getAsArray("http.netty.bind_host", settings.getAsArray("http.bind_host", settings.getAsArray("http.host", null)));
+        this.publishHosts = settings.getAsArray("http.netty.publish_host", settings.getAsArray("http.publish_host", settings.getAsArray("http.host", null)));
         this.publishPort = settings.getAsInt("http.netty.publish_port", settings.getAsInt("http.publish_port", 0));
         this.tcpNoDelay = settings.get("http.netty.tcp_no_delay", settings.get(TCP_NO_DELAY, "true"));
         this.tcpKeepAlive = settings.get("http.netty.tcp_keep_alive", settings.get(TCP_KEEP_ALIVE, "true"));
@@ -246,9 +247,9 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
         // Bind and start to accept incoming connections.
         InetAddress hostAddresses[];
         try {
-            hostAddresses = networkService.resolveBindHostAddress(bindHost);
+            hostAddresses = networkService.resolveBindHostAddresses(bindHosts);
         } catch (IOException e) {
-            throw new BindHttpException("Failed to resolve host [" + bindHost + "]", e);
+            throw new BindHttpException("Failed to resolve host [" + Arrays.toString(bindHosts) + "]", e);
         }
 
         List<InetSocketTransportAddress> boundAddresses = new ArrayList<>(hostAddresses.length);
@@ -262,7 +263,7 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
             publishPort = boundAddress.getPort();
         }
         try {
-            publishAddress = new InetSocketAddress(networkService.resolvePublishHostAddress(publishHost), publishPort);
+            publishAddress = new InetSocketAddress(networkService.resolvePublishHostAddresses(publishHosts), publishPort);
         } catch (Exception e) {
             throw new BindTransportException("Failed to resolve publish address", e);
         }

core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java

@@ -343,9 +343,9 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         return unmodifiableMap(new HashMap<>(profileBoundAddresses));
     }
 
-    private InetSocketAddress createPublishAddress(String publishHost, int publishPort) {
+    private InetSocketAddress createPublishAddress(String publishHosts[], int publishPort) {
         try {
-            return new InetSocketAddress(networkService.resolvePublishHostAddress(publishHost), publishPort);
+            return new InetSocketAddress(networkService.resolvePublishHostAddresses(publishHosts), publishPort);
         } catch (Exception e) {
             throw new BindTransportException("Failed to resolve publish address", e);
         }
@@ -436,11 +436,11 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
     private void bindServerBootstrap(final String name, final Settings settings) {
         // Bind and start to accept incoming connections.
         InetAddress hostAddresses[];
-        String bindHost = settings.get("bind_host");
+        String bindHosts[] = settings.getAsArray("bind_host", null);
         try {
-            hostAddresses = networkService.resolveBindHostAddress(bindHost);
+            hostAddresses = networkService.resolveBindHostAddresses(bindHosts);
         } catch (IOException e) {
-            throw new BindTransportException("Failed to resolve host [" + bindHost + "]", e);
+            throw new BindTransportException("Failed to resolve host " + Arrays.toString(bindHosts) + "", e);
         }
         if (logger.isDebugEnabled()) {
             String[] addresses = new String[hostAddresses.length];
@@ -493,8 +493,8 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
             if (boundTransportAddress == null) {
                 // no address is bound, so lets create one with the publish address information from the settings or the bound address as a fallback
                 int publishPort = profileSettings.getAsInt("publish_port", boundAddress.getPort());
-                String publishHost = profileSettings.get("publish_host", boundAddress.getHostString());
-                InetSocketAddress publishAddress = createPublishAddress(publishHost, publishPort);
+                String publishHosts[] = profileSettings.getAsArray("publish_host", new String[] { boundAddress.getHostString() });
+                InetSocketAddress publishAddress = createPublishAddress(publishHosts, publishPort);
                 profileBoundAddresses.put(name, new BoundTransportAddress(new TransportAddress[]{new InetSocketTransportAddress(boundAddress)}, new InetSocketTransportAddress(publishAddress)));
             } else {
                 // TODO: support real multihoming with publishing. Today we update the bound addresses so only the prioritized address is published
@@ -511,8 +511,8 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
                 // these calls are different from the profile ones due to the way the settings for a profile are created. If we want to merge the code for the default profile and
                 // other profiles together, we need to change how the profileSettings are built for the default profile...
                 int publishPort = settings.getAsInt("transport.netty.publish_port", settings.getAsInt("transport.publish_port", boundAddress.getPort()));
-                String publishHost = settings.get("transport.netty.publish_host", settings.get("transport.publish_host", settings.get("transport.host")));
-                InetSocketAddress publishAddress = createPublishAddress(publishHost, publishPort);
+                String publishHosts[] = settings.getAsArray("transport.netty.publish_host", settings.getAsArray("transport.publish_host", settings.getAsArray("transport.host", null)));
+                InetSocketAddress publishAddress = createPublishAddress(publishHosts, publishPort);
                 this.boundAddress = new BoundTransportAddress(new TransportAddress[]{new InetSocketTransportAddress(boundAddress)}, new InetSocketTransportAddress(publishAddress));
             } else {
                 // the default profile is already bound to one address and has the publish address, copy the existing bound addresses as is and append the new address.

core/src/test/java/org/elasticsearch/common/network/NetworkServiceTests.java

@@ -36,7 +36,7 @@ public class NetworkServiceTests extends ESTestCase {
     public void testBindMulticastV4() throws Exception {
         NetworkService service = new NetworkService(Settings.EMPTY);
         try {
-            service.resolveBindHostAddress("239.1.1.1");
+            service.resolveBindHostAddresses(new String[] { "239.1.1.1" });
             fail("should have hit exception");
         } catch (IllegalArgumentException e) {
             assertTrue(e.getMessage().contains("invalid: multicast"));
@@ -49,7 +49,7 @@ public class NetworkServiceTests extends ESTestCase {
     public void testBindMulticastV6() throws Exception {
         NetworkService service = new NetworkService(Settings.EMPTY);
         try {
-            service.resolveBindHostAddress("FF08::108");
+            service.resolveBindHostAddresses(new String[] { "FF08::108" });
             fail("should have hit exception");
         } catch (IllegalArgumentException e) {
             assertTrue(e.getMessage().contains("invalid: multicast"));
@@ -62,7 +62,7 @@ public class NetworkServiceTests extends ESTestCase {
     public void testPublishMulticastV4() throws Exception {
         NetworkService service = new NetworkService(Settings.EMPTY);
         try {
-            service.resolvePublishHostAddress("239.1.1.1");
+            service.resolvePublishHostAddresses(new String[] { "239.1.1.1" });
             fail("should have hit exception");
         } catch (IllegalArgumentException e) {
             assertTrue(e.getMessage().contains("invalid: multicast"));
@@ -75,7 +75,7 @@ public class NetworkServiceTests extends ESTestCase {
     public void testPublishMulticastV6() throws Exception {
         NetworkService service = new NetworkService(Settings.EMPTY);
         try {
-            service.resolvePublishHostAddress("FF08::108");
+            service.resolvePublishHostAddresses(new String[] { "FF08::108" });
             fail("should have hit exception");
         } catch (IllegalArgumentException e) {
             assertTrue(e.getMessage().contains("invalid: multicast"));
@@ -87,7 +87,7 @@ public class NetworkServiceTests extends ESTestCase {
      */
     public void testBindAnyLocalV4() throws Exception {
         NetworkService service = new NetworkService(Settings.EMPTY);
-        assertEquals(InetAddress.getByName("0.0.0.0"), service.resolveBindHostAddress("0.0.0.0")[0]);
+        assertEquals(InetAddress.getByName("0.0.0.0"), service.resolveBindHostAddresses(new String[] { "0.0.0.0" })[0]);
     }
     
     /** 
@@ -95,36 +95,24 @@ public class NetworkServiceTests extends ESTestCase {
      */
     public void testBindAnyLocalV6() throws Exception {
         NetworkService service = new NetworkService(Settings.EMPTY);
-        assertEquals(InetAddress.getByName("::"), service.resolveBindHostAddress("::")[0]);
+        assertEquals(InetAddress.getByName("::"), service.resolveBindHostAddresses(new String[] { "::" })[0]);
     }
 
     /** 
      * ensure specifying wildcard ipv4 address selects reasonable publish address 
      */
     public void testPublishAnyLocalV4() throws Exception {
-        InetAddress expected = null;
-        try {
-            expected = NetworkUtils.getFirstNonLoopbackAddresses()[0];
-        } catch (Exception e) {
-            assumeNoException("test requires up-and-running non-loopback address", e);
-        }
-        
         NetworkService service = new NetworkService(Settings.EMPTY);
-        assertEquals(expected, service.resolvePublishHostAddress("0.0.0.0"));
+        InetAddress address = service.resolvePublishHostAddresses(new String[] { "0.0.0.0" });
+        assertFalse(address.isAnyLocalAddress());
     }
 
     /** 
      * ensure specifying wildcard ipv6 address selects reasonable publish address 
      */
     public void testPublishAnyLocalV6() throws Exception {
-        InetAddress expected = null;
-        try {
-            expected = NetworkUtils.getFirstNonLoopbackAddresses()[0];
-        } catch (Exception e) {
-            assumeNoException("test requires up-and-running non-loopback address", e);
-        }
-        
         NetworkService service = new NetworkService(Settings.EMPTY);
-        assertEquals(expected, service.resolvePublishHostAddress("::"));
+        InetAddress address = service.resolvePublishHostAddresses(new String[] { "::" });
+        assertFalse(address.isAnyLocalAddress());
     }
 }

docs/reference/migration/migrate_3_0.asciidoc

@@ -329,3 +329,9 @@ to index a document only if it doesn't already exist.
 Two cache concurrency level settings `indices.requests.cache.concurrency_level` and
 `indices.fielddata.cache.concurrency_level` because they no longer apply to the cache implementation used for the
 request cache and the field data cache.
+
+=== Remove bind option of `non_loopback`
+
+This setting would arbitrarily pick the first interface not marked as loopback. Instead, specify by address
+scope (e.g. `_local_,_site_` for all loopback and private network addresses) or by explicit interface names,
+hostnames, or addresses. 

docs/reference/modules/network.asciidoc

@@ -8,6 +8,9 @@ configuration, for example, the
 network settings allows to set common settings that will be shared among
 all network based modules (unless explicitly overridden in each module).
 
+Be careful with host configuration! Never expose an unprotected instance
+to the public internet.
+
 The `network.bind_host` setting allows to control the host different network
 components will bind on. By default, the bind host will be `_local_`
 (loopback addresses such as `127.0.0.1`, `::1`).
@@ -16,14 +19,13 @@ The `network.publish_host` setting allows to control the host the node will
 publish itself within the cluster so other nodes will be able to connect to it.
 Currently an elasticsearch node may be bound to multiple addresses, but only
 publishes one.  If not specified, this defaults to the "best" address from 
-`network.bind_host`.  By default, IPv4 addresses are preferred to IPv6, and 
-ordinary addresses are preferred to site-local or link-local addresses.
+`network.bind_host`, sorted by IPv4/IPv6 stack preference, then by reachability.
 
 The `network.host` setting is a simple setting to automatically set both
 `network.bind_host` and `network.publish_host` to the same host value.
 
-Both settings allows to be configured with either explicit host address
-or host name. The settings also accept logical setting values explained
+Both settings allows to be configured with either explicit host address(es)
+or host name(s). The settings also accept logical setting value(s) explained
 in the following table:
 
 [cols="<,<",options="header",]
@@ -31,15 +33,21 @@ in the following table:
 |Logical Host Setting Value |Description
 |`_local_` |Will be resolved to loopback addresses
 
-|`_local:ipv4_` |Will be resolved to loopback IPv4 addresses
+|`_local:ipv4_` |Will be resolved to loopback IPv4 addresses (e.g. 127.0.0.1)
+
+|`_local:ipv6_` |Will be resolved to loopback IPv6 addresses (e.g. ::1)
+
+|`_site_` |Will be resolved to site-local addresses ("private network")
+
+|`_site:ipv4_` |Will be resolved to site-local IPv4 addresses (e.g. 192.168.0.1)
 
-|`_local:ipv6_` |Will be resolved to loopback IPv6 addresses
+|`_site:ipv6_` |Will be resolved to site-local IPv6 addresses (e.g. fec0::1)
 
-|`_non_loopback_` |Addresses of the first non loopback interface
+|`_global_` |Will be resolved to globally-scoped addresses ("publicly reachable")
 
-|`_non_loopback:ipv4_` |IPv4 addresses of the first non loopback interface
+|`_global:ipv4_` |Will be resolved to globally-scoped IPv4 addresses (e.g. 8.8.8.8)
 
-|`_non_loopback:ipv6_` |IPv6 addresses of the first non loopback interface
+|`_global:ipv6_` |Will be resolved to globally-scoped IPv6 addresses (e.g. 2001:4860:4860::8888)
 
 |`_[networkInterface]_` |Resolves to the addresses of the provided
 network interface. For example `_en0_`.

plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java

@@ -177,7 +177,7 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic
 
         InetAddress ipAddress = null;
         try {
-            ipAddress = networkService.resolvePublishHostAddress(null);
+            ipAddress = networkService.resolvePublishHostAddresses(null);
             logger.trace("ip of current node: [{}]", ipAddress);
         } catch (IOException e) {
             // We can't find the publish host address... Hmmm. Too bad :-(

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java

@@ -46,7 +46,7 @@ public class Ec2NetworkTests extends ESTestCase {
         networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings));
         // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
         try {
-            networkService.resolveBindHostAddress(null);
+            networkService.resolveBindHostAddresses(null);
         } catch (IOException e) {
             assertThat(e.getMessage(), containsString("local-ipv4"));
         }
@@ -64,7 +64,7 @@ public class Ec2NetworkTests extends ESTestCase {
         networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings));
         // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
         try {
-            networkService.resolveBindHostAddress(null);
+            networkService.resolveBindHostAddresses(null);
         } catch (IOException e) {
             assertThat(e.getMessage(), containsString("public-ipv4"));
         }
@@ -82,7 +82,7 @@ public class Ec2NetworkTests extends ESTestCase {
         networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings));
         // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
         try {
-            networkService.resolveBindHostAddress(null);
+            networkService.resolveBindHostAddresses(null);
         } catch (IOException e) {
             assertThat(e.getMessage(), containsString("local-ipv4"));
         }
@@ -100,7 +100,7 @@ public class Ec2NetworkTests extends ESTestCase {
         networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings));
         // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
         try {
-            networkService.resolveBindHostAddress(null);
+            networkService.resolveBindHostAddresses(null);
         } catch (IOException e) {
             assertThat(e.getMessage(), containsString("local-ipv4"));
         }
@@ -118,7 +118,7 @@ public class Ec2NetworkTests extends ESTestCase {
         networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings));
         // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
         try {
-            networkService.resolveBindHostAddress(null);
+            networkService.resolveBindHostAddresses(null);
         } catch (IOException e) {
             assertThat(e.getMessage(), containsString("local-hostname"));
         }
@@ -136,7 +136,7 @@ public class Ec2NetworkTests extends ESTestCase {
         networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings));
         // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
         try {
-            networkService.resolveBindHostAddress(null);
+            networkService.resolveBindHostAddresses(null);
         } catch (IOException e) {
             assertThat(e.getMessage(), containsString("public-ipv4"));
         }
@@ -154,7 +154,7 @@ public class Ec2NetworkTests extends ESTestCase {
         networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings));
         // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
         try {
-            networkService.resolveBindHostAddress(null);
+            networkService.resolveBindHostAddresses(null);
         } catch (IOException e) {
             assertThat(e.getMessage(), containsString("public-hostname"));
         }
@@ -171,7 +171,7 @@ public class Ec2NetworkTests extends ESTestCase {
 
         NetworkService networkService = new NetworkService(nodeSettings);
         networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings));
-        InetAddress[] addresses = networkService.resolveBindHostAddress(null);
-        assertThat(addresses, arrayContaining(networkService.resolveBindHostAddress("_local_")));
+        InetAddress[] addresses = networkService.resolveBindHostAddresses(null);
+        assertThat(addresses, arrayContaining(networkService.resolveBindHostAddresses(new String[] { "_local_" })));
     }
 }

plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java

@@ -110,7 +110,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas
         cachedDiscoNodes = new ArrayList<>();
         String ipAddress = null;
         try {
-            InetAddress inetAddress = networkService.resolvePublishHostAddress(null);
+            InetAddress inetAddress = networkService.resolvePublishHostAddresses(null);
             if (inetAddress != null) {
                 ipAddress = NetworkAddress.formatAddress(inetAddress);
             }

plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java

@@ -78,8 +78,8 @@ public class GceNetworkTests extends ESTestCase {
      * Test that we don't have any regression with network host core settings such as
      * network.host: _local_
      */
-    public void testNetworkHostCoreLocal() throws IOException {
-        resolveGce("_local_", new NetworkService(Settings.EMPTY).resolveBindHostAddress(NetworkService.DEFAULT_NETWORK_HOST));
+    public void networkHostCoreLocal() throws IOException {
+        resolveGce("_local_", new NetworkService(Settings.EMPTY).resolveBindHostAddresses(new String[] { NetworkService.DEFAULT_NETWORK_HOST }));
     }
 
     /**
@@ -107,7 +107,7 @@ public class GceNetworkTests extends ESTestCase {
         GceComputeServiceMock mock = new GceComputeServiceMock(nodeSettings, networkService);
         networkService.addCustomNameResolver(new GceNameResolver(nodeSettings, mock));
         try {
-            InetAddress[] addresses = networkService.resolveBindHostAddress(null);
+            InetAddress[] addresses = networkService.resolveBindHostAddresses(null);
             if (expected == null) {
                 fail("We should get a IllegalArgumentException when setting network.host: _gce:doesnotexist_");
             }

plugins/discovery-multicast/pom.xml

@@ -19,6 +19,7 @@
         <tests.jvms>1</tests.jvms>
         <tests.rest.suite>discovery_multicast</tests.rest.suite>
         <tests.rest.load_packaged>false</tests.rest.load_packaged>
+        <xlint.options>-Xlint:-deprecation</xlint.options>
     </properties>
 
     <build>

plugins/discovery-multicast/src/main/java/org/elasticsearch/plugin/discovery/multicast/MulticastZenPing.java

@@ -35,6 +35,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
@@ -55,7 +56,10 @@ import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.net.SocketAddress;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -138,11 +142,14 @@ public class MulticastZenPing extends AbstractLifecycleComponent<ZenPing> implem
             boolean shared = settings.getAsBoolean("discovery.zen.ping.multicast.shared", Constants.MAC_OS_X);
             // OSX does not correctly send multicasts FROM the right interface
             boolean deferToInterface = settings.getAsBoolean("discovery.zen.ping.multicast.defer_group_to_set_interface", Constants.MAC_OS_X);
+            // don't use publish address, the use case for that is e.g. a firewall or proxy and
+            // may not even be bound to an interface on this machine! use the first bound address.
+            List<InetAddress> addresses = Arrays.asList(networkService.resolveBindHostAddresses(address == null ? null : new String[] { address }));
+            NetworkUtils.sortAddresses(addresses);
+            
             multicastChannel = MulticastChannel.getChannel(nodeName(), shared,
                     new MulticastChannel.Config(port, group, bufferSize, ttl,
-                            // don't use publish address, the use case for that is e.g. a firewall or proxy and
-                            // may not even be bound to an interface on this machine! use the first bound address.
-                            networkService.resolveBindHostAddress(address)[0],
+                            addresses.get(0),
                             deferToInterface),
                     new Receiver());
         } catch (Throwable t) {