Update GeoIP database service URL (#69862)

This change updates GeoIP database service URL to the new https://geoip.elastic.co/v1/database and removes (now optional) key/UUID parameter.
It also fixes geoip-fixture to provide 3 different test databases (City, Country and ASN).
It also unmutes GeoIpDownloaderIT. testGeoIpDatabasesDownload with additional logging and increased timeouts which tries to address #69594

modules/ingest-geoip/build.gradle

@@ -42,7 +42,7 @@ def fixtureAddress = {
   assert useFixture: 'closure should not be used without a fixture'
   int ephemeralPort = tasks.getByPath(":test:fixtures:geoip-fixture:postProcessFixture").ext."test.fixtures.geoip-fixture.tcp.80"
   assert ephemeralPort > 0
-  return "http://127.0.0.1:${ephemeralPort}"
+  return "http://127.0.0.1:${ephemeralPort}/"
 }
 
 if (useFixture) {

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java

@@ -75,7 +75,6 @@ public class GeoIpDownloaderIT extends AbstractGeoIpIT {
         assertTrue(settingsResponse.isAcknowledged());
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/69594")
     public void testGeoIpDatabasesDownload() throws Exception {
         // use short wait for local fixture, longer when we hit real service
         int waitTime = ENDPOINT == null ? 120 : 10;

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java

@@ -60,7 +60,7 @@ class GeoIpDownloader extends AllocatedPersistentTask {
     public static final Setting<TimeValue> POLL_INTERVAL_SETTING = Setting.timeSetting("geoip.downloader.poll.interval",
         TimeValue.timeValueDays(3), TimeValue.timeValueDays(1), Property.Dynamic, Property.NodeScope);
     public static final Setting<String> ENDPOINT_SETTING = Setting.simpleString("geoip.downloader.endpoint",
-        "https://paisano.elastic.dev/v1/geoip/database", Property.NodeScope);
+        "https://geoip.elastic.co/v1/database", Property.NodeScope);
 
     public static final String GEOIP_DOWNLOADER = "geoip-downloader";
     static final String DATABASES_INDEX = ".geoip_databases";
@@ -106,7 +106,9 @@ class GeoIpDownloader extends AllocatedPersistentTask {
 
     @SuppressWarnings("unchecked")
     private <T> List<T> fetchDatabasesOverview() throws IOException {
-        byte[] data = httpClient.getBytes(endpoint + "?key=11111111-1111-1111-1111-111111111111&elastic_geoip_service_tos=agree");
+        String url = endpoint + "?elastic_geoip_service_tos=agree";
+        logger.info("fetching geoip databases overview from [" + url + "]");
+        byte[] data = httpClient.getBytes(url);
         try (XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
             DeprecationHandler.THROW_UNSUPPORTED_OPERATION, data)) {
             return (List<T>) parser.list();

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java

@@ -75,8 +75,8 @@ class HttpClient {
 
     private HttpURLConnection createConnection(String url) throws IOException {
         HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
-        conn.setConnectTimeout(5000);
-        conn.setReadTimeout(5000);
+        conn.setConnectTimeout(10000);
+        conn.setReadTimeout(10000);
         conn.setDoOutput(false);
         conn.setInstanceFollowRedirects(false);
         return conn;

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java

@@ -323,7 +323,7 @@ public class GeoIpDownloaderTests extends ESTestCase {
         builder.map(Map.of("a", 2));
         builder.endArray();
         builder.close();
-        when(httpClient.getBytes("a.b?key=11111111-1111-1111-1111-111111111111&elastic_geoip_service_tos=agree"))
+        when(httpClient.getBytes("a.b?elastic_geoip_service_tos=agree"))
             .thenReturn(baos.toByteArray());
         Iterator<Map<String, Object>> it = maps.iterator();
         geoIpDownloader = new GeoIpDownloader(client, httpClient, clusterService, threadPool,

test/fixtures/geoip-fixture/src/main/java/fixture/geoip/GeoIpHttpFixture.java

@@ -11,6 +11,7 @@ package fixture.geoip;
 import com.sun.net.httpserver.HttpServer;
 
 import java.io.BufferedWriter;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.net.InetAddress;
@@ -26,8 +27,9 @@ public class GeoIpHttpFixture {
         this.server = HttpServer.create(new InetSocketAddress(InetAddress.getByName(args[0]), Integer.parseInt(args[1])), 0);
         this.server.createContext("/", exchange -> {
             String query = exchange.getRequestURI().getQuery();
-            if (query.contains("elastic_geoip_service_tos=agree") == false) {
+            if (query == null || query.contains("elastic_geoip_service_tos=agree") == false) {
                 exchange.sendResponseHeaders(400, 0);
+                exchange.getResponseBody().close();
                 return;
             }
             String data = rawData.replace("endpoint", "http://" + exchange.getRequestHeaders().getFirst("Host"));
@@ -36,10 +38,12 @@ public class GeoIpHttpFixture {
                 writer.write(data);
             }
         });
-        this.server.createContext("/db.mmdb.gz", exchange -> {
+        this.server.createContext("/db", exchange -> {
             exchange.sendResponseHeaders(200, 0);
-            try (OutputStream outputStream = exchange.getResponseBody()) {
-                GeoIpHttpFixture.class.getResourceAsStream("/GeoIP2-City-Test.mmdb.gz").transferTo(outputStream);
+            String dbName = exchange.getRequestURI().getPath().replaceAll(".*/db", "");
+            try (OutputStream outputStream = exchange.getResponseBody();
+                 InputStream db = GeoIpHttpFixture.class.getResourceAsStream(dbName)) {
+                db.transferTo(outputStream);
             }
         });
     }

test/fixtures/geoip-fixture/src/main/resources/data.json

@@ -1,20 +1,20 @@
 [
   {
-    "md5_hash": "dd3ac893b3b858d8f45674345dfe6fe9",
+    "md5_hash": "a7813f5053276eb22639b61c61e3e56c",
     "name": "GeoLite2-City.mmdb.gz",
-    "url": "endpoint/db.mmdb.gz",
+    "url": "endpoint/db/GeoLite2-City.mmdb.gz",
     "provider": "maxmind"
   },
   {
-    "md5_hash": "dd3ac893b3b858d8f45674345dfe6fe9",
+    "md5_hash": "f452b1aece8dc6137485ddbb0401d6fe",
     "name": "GeoLite2-ASN.mmdb.gz",
-    "url": "endpoint/db.mmdb.gz",
+    "url": "endpoint/db/GeoLite2-ASN.mmdb.gz",
     "provider": "maxmind"
   },
   {
-    "md5_hash": "dd3ac893b3b858d8f45674345dfe6fe9",
+    "md5_hash": "dc366cf57a4f101d722b4cd10d4a9825",
     "name": "GeoLite2-Country.mmdb.gz",
-    "url": "endpoint/db.mmdb.gz",
+    "url": "endpoint/db/GeoLite2-Country.mmdb.gz",
     "provider": "maxmind"
   }
 ]