Use hostname instead of IP with SPNEGO test (#32514)

This change updates KerberosAuthenticationIT to resolve the host used
to connect to the test cluster. This is needed because the host could
be an IP address but SPNEGO requires a hostname to work properly. This
is done by adding a hook in ESRestTestCase for building the HttpHost
from the host and port.

Additionally, the project now specifies the IPv4 loopback address as
the http host. This is done because we need to be able to resolve the
address used for the HTTP transport before the node starts up, but the
http.ports file is not written until the node is started.

Closes #32498

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -148,7 +148,7 @@ public abstract class ESRestTestCase extends ESTestCase {
                 }
                 String host = stringUrl.substring(0, portSeparator);
                 int port = Integer.valueOf(stringUrl.substring(portSeparator + 1));
-                hosts.add(new HttpHost(host, port, getProtocol()));
+                hosts.add(buildHttpHost(host, port));
             }
             clusterHosts = unmodifiableList(hosts);
             logger.info("initializing REST clients against {}", clusterHosts);
@@ -160,6 +160,13 @@ public abstract class ESRestTestCase extends ESTestCase {
         assert clusterHosts != null;
     }
 
+    /**
+     * Construct a HttpHost from the given host and port
+     */
+    protected HttpHost buildHttpHost(String host, int port) {
+        return new HttpHost(host, port, getProtocol());
+    }
+
     /**
      * Clean up after the test case.
      */

x-pack/qa/kerberos-tests/build.gradle

@@ -36,32 +36,43 @@ task krb5kdcFixture(type: org.elasticsearch.gradle.test.VagrantFixture) {
     dependsOn krb5kdcUpdate
 }
 
-task krb5AddPrincipals { dependsOn krb5kdcFixture }
+// lazily resolve to avoid any slowdowns from DNS lookups prior to when we need this value
+Object httpPrincipal = new Object() {
+    @Override
+    String toString() {
+        InetAddress resolvedAddress = InetAddress.getByName('127.0.0.1')
+        return "HTTP/" + resolvedAddress.getHostName()
+    }
+}
 
-List<String> principals = [
-    "HTTP/localhost",
-    "peppa",
-    "george~dino"
-]
 String realm = "BUILD.ELASTIC.CO"
 
-for (String principal : principals) {
-    String[] princPwdPair = principal.split('~');
-    String princName = princPwdPair[0];
-    String password = "";
-    if (princPwdPair.length > 1) {
-        password = princPwdPair[1];
-    }
-    Task create = project.tasks.create("addPrincipal#${principal}".replace('/', '_'), org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
-        command 'ssh'
-        args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh $princName $password"
-        boxName box
-        environmentVars vagrantEnvVars
-        dependsOn krb5kdcFixture
-    }
-    krb5AddPrincipals.dependsOn(create)
+task 'addPrincipal#peppa'(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'ssh'
+    args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh peppa "
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcFixture
+}
+
+task 'addPrincipal#george'(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'ssh'
+    args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh george dino"
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcFixture
+}
+
+task 'addPrincipal#HTTP'(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'ssh'
+    args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh $httpPrincipal"
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcFixture
 }
 
+task krb5AddPrincipals { dependsOn krb5kdcFixture, 'addPrincipal#peppa', 'addPrincipal#george', 'addPrincipal#HTTP' }
+
 def generatedResources = "$buildDir/generated-resources/keytabs"
 task copyKeytabToGeneratedResources(type: Copy) {
     Path peppaKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("peppa.keytab").toAbsolutePath()
@@ -71,6 +82,9 @@ task copyKeytabToGeneratedResources(type: Copy) {
 }
 
 integTestCluster {
+    // force localhost IPv4 otherwise it is a chicken and egg problem where we need the keytab for the hostname when starting the cluster
+    // but do not know the exact address that is first in the http ports file
+    setting 'http.host', '127.0.0.1'
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.security.authc.realms.file.type', 'file'
@@ -87,7 +101,8 @@ integTestCluster {
     Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
     String jvmArgsStr = " -Djava.security.krb5.conf=${krb5conf}" + " -Dsun.security.krb5.debug=true"
     jvmArgs jvmArgsStr
-    Path esKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("HTTP_localhost.keytab").toAbsolutePath()
+    Path esKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs")
+            .resolve("$httpPrincipal".replace('/', '_') + ".keytab").toAbsolutePath()
     extraConfigFile("es.keytab", "${esKeytab}")
 
     setupCommand 'setupTestAdmin',

x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java

@@ -13,6 +13,7 @@ import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -23,6 +24,8 @@ import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.security.AccessControlContext;
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
@@ -82,7 +85,6 @@ public class KerberosAuthenticationIT extends ESRestTestCase {
         assertOK(response);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32498")
     public void testLoginByKeytab() throws IOException, PrivilegedActionException {
         final String userPrincipalName = System.getProperty(TEST_USER_WITH_KEYTAB_KEY);
         final String keytabPath = System.getProperty(TEST_USER_WITH_KEYTAB_PATH_KEY);
@@ -92,7 +94,6 @@ public class KerberosAuthenticationIT extends ESRestTestCase {
         executeRequestAndVerifyResponse(userPrincipalName, callbackHandler);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32498")
     public void testLoginByUsernamePassword() throws IOException, PrivilegedActionException {
         final String userPrincipalName = System.getProperty(TEST_USER_WITH_PWD_KEY);
         final String password = System.getProperty(TEST_USER_WITH_PWD_PASSWD_KEY);
@@ -106,6 +107,18 @@ public class KerberosAuthenticationIT extends ESRestTestCase {
         // intentionally empty - this is just needed to ensure the build does not fail because we mute its only test.
     }
 
+    @Override
+    @SuppressForbidden(reason = "SPNEGO relies on hostnames and we need to ensure host isn't a IP address")
+    protected HttpHost buildHttpHost(String host, int port) {
+        try {
+            InetAddress inetAddress = InetAddress.getByName(host);
+            return super.buildHttpHost(inetAddress.getHostName(), port);
+        } catch (UnknownHostException e) {
+            assumeNoException("failed to resolve host [" + host + "]", e);
+        }
+        throw new IllegalStateException("DNS not resolved and assume did not trip");
+    }
+
     private void executeRequestAndVerifyResponse(final String userPrincipalName,
             final SpnegoHttpClientConfigCallbackHandler callbackHandler) throws PrivilegedActionException, IOException {
         final Request request = new Request("GET", "/_xpack/security/_authenticate");