update http client version to 4.5.2 and http-core 4.4.4 (#18399)

StrictHostnameVerifier can now be removed

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -1335,7 +1335,6 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]junit[/\\]listeners[/\\]LoggingListener.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]ESRestTestCase.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]RestTestExecutionContext.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]client[/\\]RestClient.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]client[/\\]http[/\\]HttpRequestBuilder.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]json[/\\]JsonPath.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]GreaterThanEqualToParser.java" checks="LineLength" />

buildSrc/version.properties

@@ -13,9 +13,7 @@ jna               = 4.1.0
 # test dependencies
 randomizedrunner  = 2.3.2
 junit             = 4.11
-# TODO: Upgrade httpclient to a version > 4.5.1 once released. Then remove o.e.test.rest.client.StrictHostnameVerifier* and use
-#  DefaultHostnameVerifier instead since we no longer need to workaround https://issues.apache.org/jira/browse/HTTPCLIENT-1698
-httpclient        = 4.3.6
-httpcore          = 4.3.3
+httpclient        = 4.5.2
+httpcore          = 4.4.4
 commonslogging    = 1.1.3
 commonscodec      = 1.10

test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java

@@ -19,16 +19,15 @@
 package org.elasticsearch.test.rest.client;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
-
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
 import org.apache.http.conn.socket.PlainConnectionSocketFactory;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLContexts;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContexts;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
@@ -134,7 +133,8 @@ public class RestClient implements Closeable {
      * @throws RestException if the obtained status code is non ok, unless the specific error code needs to be ignored
      * according to the ignore parameter received as input (which won't get sent to elasticsearch)
      */
-    public RestResponse callApi(String apiName, Map<String, String> params, String body, Map<String, String> headers) throws IOException, RestException {
+    public RestResponse callApi(String apiName, Map<String, String> params, String body, Map<String, String> headers)
+            throws IOException, RestException {
 
         List<Integer> ignores = new ArrayList<>();
         Map<String, String> requestParams = null;
@@ -220,7 +220,8 @@ public class RestClient implements Closeable {
                     if (restApi.getParams().contains(entry.getKey()) || ALWAYS_ACCEPTED_QUERY_STRING_PARAMS.contains(entry.getKey())) {
                         httpRequestBuilder.addParam(entry.getKey(), entry.getValue());
                     } else {
-                        throw new IllegalArgumentException("param [" + entry.getKey() + "] not supported in [" + restApi.getName() + "] api");
+                        throw new IllegalArgumentException("param [" + entry.getKey() +
+                                "] not supported in [" + restApi.getName() + "] api");
                     }
                 }
             }
@@ -293,10 +294,8 @@ public class RestClient implements Closeable {
                 try (InputStream is = Files.newInputStream(path)) {
                     keyStore.load(is, keystorePass.toCharArray());
                 }
-                SSLContext sslcontext = SSLContexts.custom()
-                        .loadTrustMaterial(keyStore, null)
-                        .build();
-                sslsf = new SSLConnectionSocketFactory(sslcontext, StrictHostnameVerifier.INSTANCE);
+                SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(keyStore, null).build();
+                sslsf = new SSLConnectionSocketFactory(sslcontext);
             } catch (KeyStoreException|NoSuchAlgorithmException|KeyManagementException|CertificateException e) {
                 throw new RuntimeException(e);
             }
@@ -308,7 +307,8 @@ public class RestClient implements Closeable {
                 .register("http", PlainConnectionSocketFactory.getSocketFactory())
                 .register("https", sslsf)
                 .build();
-        return HttpClients.createMinimal(new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, 15, TimeUnit.SECONDS));
+        return HttpClients.createMinimal(
+                new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, 15, TimeUnit.SECONDS));
     }
 
     /**

test/framework/src/main/java/org/elasticsearch/test/rest/client/StrictHostnameVerifier.java

@@ -1,76 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.client;
-
-import org.apache.http.conn.ssl.X509HostnameVerifier;
-import org.apache.http.conn.util.InetAddressUtils;
-
-import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.SSLSocket;
-import java.io.IOException;
-import java.security.cert.X509Certificate;
-
-/**
- * A custom {@link X509HostnameVerifier} implementation that wraps calls to the {@link org.apache.http.conn.ssl.StrictHostnameVerifier} and
- * properly handles IPv6 addresses that come from a URL in the form <code>http://[::1]:9200/</code> by removing the surrounding brackets.
- *
- * This is a variation of the fix for <a href="https://issues.apache.org/jira/browse/HTTPCLIENT-1698">HTTPCLIENT-1698</a>, which is not
- * released yet as of Apache HttpClient 4.5.1
- */
-final class StrictHostnameVerifier implements X509HostnameVerifier {
-
-    static final StrictHostnameVerifier INSTANCE = new StrictHostnameVerifier();
-
-    // We need to wrap the default verifier for HttpClient since we use an older version and the following issue is not
-    // fixed in a released version yet https://issues.apache.org/jira/browse/HTTPCLIENT-1698
-    // TL;DR we need to strip '[' and ']' from IPv6 addresses if they come from a URL
-    private final X509HostnameVerifier verifier = new org.apache.http.conn.ssl.StrictHostnameVerifier();
-
-    private StrictHostnameVerifier() {}
-
-    @Override
-    public boolean verify(String host, SSLSession sslSession) {
-        return verifier.verify(stripBracketsIfNecessary(host), sslSession);
-    }
-
-    @Override
-    public void verify(String host, SSLSocket ssl) throws IOException {
-        verifier.verify(stripBracketsIfNecessary(host), ssl);
-    }
-
-    @Override
-    public void verify(String host, X509Certificate cert) throws SSLException {
-        verifier.verify(stripBracketsIfNecessary(host), cert);
-    }
-
-    @Override
-    public void verify(String host, String[] cns, String[] subjectAlts) throws SSLException {
-        verifier.verify(stripBracketsIfNecessary(host), cns, subjectAlts);
-    }
-
-    private String stripBracketsIfNecessary(String host) {
-        if (host.startsWith("[") && host.endsWith("]")) {
-            String newHost = host.substring(1, host.length() - 1);
-            assert InetAddressUtils.isIPv6Address(newHost);
-            return newHost;
-        }
-        return host;
-    }
-}

test/framework/src/main/java/org/elasticsearch/test/rest/client/StrictHostnameVerifierTests.java

@@ -1,120 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.client;
-
-import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
-
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.SSLSocket;
-import javax.security.auth.x500.X500Principal;
-import java.security.cert.Certificate;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Tests for the {@link StrictHostnameVerifier} to validate that it can verify IPv6 addresses with and without bracket notation, in
- * addition to other address types.
- */
-public class StrictHostnameVerifierTests extends ESTestCase {
-
-    private static final int IP_SAN_TYPE = 7;
-    private static final int DNS_SAN_TYPE = 2;
-
-    private static final String[] CNS = new String[] { "my node" };
-    private static final String[] IP_SANS = new String[] { "127.0.0.1", "192.168.1.1", "::1" };
-    private static final String[] DNS_SANS = new String[] { "localhost", "computer", "localhost6" };
-
-    private SSLSocket sslSocket;
-    private SSLSession sslSession;
-    private X509Certificate certificate;
-
-    @Before
-    public void setupMocks() throws Exception {
-        sslSocket = mock(SSLSocket.class);
-        sslSession = mock(SSLSession.class);
-        certificate = mock(X509Certificate.class);
-        Collection<List<?>> subjectAlternativeNames = new ArrayList<>();
-        for (String san : IP_SANS) {
-            subjectAlternativeNames.add(Arrays.asList(IP_SAN_TYPE, san));
-        }
-        for (String san : DNS_SANS) {
-            subjectAlternativeNames.add(Arrays.asList(DNS_SAN_TYPE, san));
-        }
-
-        when(sslSocket.getSession()).thenReturn(sslSession);
-        when(sslSession.getPeerCertificates()).thenReturn(new Certificate[] { certificate });
-        when(certificate.getSubjectX500Principal()).thenReturn(new X500Principal("CN=" + CNS[0]));
-        when(certificate.getSubjectAlternativeNames()).thenReturn(subjectAlternativeNames);
-    }
-
-    public void testThatIPv6WithBracketsWorks() throws Exception {
-        final String ipv6Host = "[::1]";
-
-        // an exception will be thrown if verification fails
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, CNS, IP_SANS);
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, sslSocket);
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, certificate);
-
-        // this is the only one we can assert on
-        assertTrue(StrictHostnameVerifier.INSTANCE.verify(ipv6Host, sslSession));
-    }
-
-    public void testThatIPV6WithoutBracketWorks() throws Exception {
-        final String ipv6Host = "::1";
-
-        // an exception will be thrown if verification fails
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, CNS, IP_SANS);
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, sslSocket);
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, certificate);
-
-        // this is the only one we can assert on
-        assertTrue(StrictHostnameVerifier.INSTANCE.verify(ipv6Host, sslSession));
-    }
-
-    public void testThatIPV4Works() throws Exception {
-        final String ipv4Host = randomFrom("127.0.0.1", "192.168.1.1");
-
-        // an exception will be thrown if verification fails
-        StrictHostnameVerifier.INSTANCE.verify(ipv4Host, CNS, IP_SANS);
-        StrictHostnameVerifier.INSTANCE.verify(ipv4Host, sslSocket);
-        StrictHostnameVerifier.INSTANCE.verify(ipv4Host, certificate);
-
-        // this is the only one we can assert on
-        assertTrue(StrictHostnameVerifier.INSTANCE.verify(ipv4Host, sslSession));
-    }
-
-    public void testThatHostnameWorks() throws Exception {
-        final String host = randomFrom(DNS_SANS);
-
-        // an exception will be thrown if verification fails
-        StrictHostnameVerifier.INSTANCE.verify(host, CNS, DNS_SANS);
-        StrictHostnameVerifier.INSTANCE.verify(host, sslSocket);
-        StrictHostnameVerifier.INSTANCE.verify(host, certificate);
-
-        // this is the only one we can assert on
-        assertTrue(StrictHostnameVerifier.INSTANCE.verify(host, sslSession));
-    }
-}