Add support for the 'ISP' database to the geoip processor (#108651)

docs/changelog/108651.yaml

@@ -0,0 +1,29 @@
+pr: 108651
+summary: Add support for the 'ISP' database to the geoip processor
+area: Ingest Node
+type: enhancement
+issues: []
+highlight:
+  title: Add support for the 'ISP' database to the geoip processor
+  body: |-
+    Follow on to https://github.com/elastic/elasticsearch/pull/107287,
+    https://github.com/elastic/elasticsearch/pull/107377, and
+    https://github.com/elastic/elasticsearch/pull/108639
+
+    Adds support for the ['GeoIP2
+    ISP'](https://dev.maxmind.com/geoip/docs/databases/isp) database from
+    MaxMind to the geoip processor.
+
+    The geoip processor will automatically download the [various 'GeoLite2'
+    databases](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data),
+    but the 'GeoIP2 ISP' database is not a 'GeoLite2' database -- it's a
+    commercial database available to those with a suitable license from
+    MaxMind.
+
+    The support that is being added for it in this PR is in line with the
+    support that we already have for MaxMind's 'GeoIP2 City' and 'GeoIP2
+    Country' databases -- that is, one would need to arrange their own
+    download management via some custom endpoint or otherwise arrange for
+    the relevant file(s) to be in the $ES_CONFIG/ingest-geoip directory on
+    the nodes of the cluster.
+  notable: true

docs/reference/ingest/processors/geoip.asciidoc

@@ -60,10 +60,15 @@ in `properties`.
 `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`. The fields actually added
 depend on what has been found and which properties were configured in `properties`.
 * If the GeoIP2 Domain database is used, then the following fields may be added under the `target_field`: `ip`, and `domain`.
+The fields actually added depend on what has been found and which properties were configured in `properties`.
+* If the GeoIP2 ISP database is used, then the following fields may be added under the `target_field`: `ip`, `asn`,
+`organization_name`, `network`, `isp`, `isp_organization`, `mobile_country_code`, and `mobile_network_code`. The fields actually added
+depend on what has been found and which properties were configured in `properties`.
 * If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
 `country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`,
-`organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`.
-The fields actually added depend on what has been found and which properties were configured in `properties`.
+`organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, `residential_proxy`,
+`isp`, `isp_organization`, `mobile_country_code`, and `mobile_network_code`. The fields actually added depend on what has been found
+and which properties were configured in `properties`.
 
 
 Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java

@@ -96,7 +96,11 @@ enum Database {
             Property.ANONYMOUS,
             Property.PUBLIC_PROXY,
             Property.RESIDENTIAL_PROXY,
-            Property.DOMAIN
+            Property.DOMAIN,
+            Property.ISP,
+            Property.ISP_ORGANIZATION_NAME,
+            Property.MOBILE_COUNTRY_CODE,
+            Property.MOBILE_NETWORK_CODE
         ),
         Set.of(
             Property.COUNTRY_ISO_CODE,
@@ -107,6 +111,28 @@ enum Database {
             Property.CITY_NAME,
             Property.LOCATION
         )
+    ),
+    Isp(
+        Set.of(
+            Property.IP,
+            Property.ASN,
+            Property.ORGANIZATION_NAME,
+            Property.NETWORK,
+            Property.ISP,
+            Property.ISP_ORGANIZATION_NAME,
+            Property.MOBILE_COUNTRY_CODE,
+            Property.MOBILE_NETWORK_CODE
+        ),
+        Set.of(
+            Property.IP,
+            Property.ASN,
+            Property.ORGANIZATION_NAME,
+            Property.NETWORK,
+            Property.ISP,
+            Property.ISP_ORGANIZATION_NAME,
+            Property.MOBILE_COUNTRY_CODE,
+            Property.MOBILE_NETWORK_CODE
+        )
     );
 
     private static final String CITY_DB_SUFFIX = "-City";
@@ -115,6 +141,7 @@ enum Database {
     private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
     private static final String DOMAIN_DB_SUFFIX = "-Domain";
     private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
+    private static final String ISP_DB_SUFFIX = "-ISP";
 
     /**
      * Parses the passed-in databaseType (presumably from the passed-in databaseFile) and return the Database instance that is
@@ -140,6 +167,8 @@ enum Database {
                 database = Database.Domain;
             } else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) {
                 database = Database.Enterprise;
+            } else if (databaseType.endsWith(Database.ISP_DB_SUFFIX)) {
+                database = Database.Isp;
             }
         }
 
@@ -215,7 +244,11 @@ enum Database {
         ANONYMOUS,
         PUBLIC_PROXY,
         RESIDENTIAL_PROXY,
-        DOMAIN;
+        DOMAIN,
+        ISP,
+        ISP_ORGANIZATION_NAME,
+        MOBILE_COUNTRY_CODE,
+        MOBILE_NETWORK_CODE;
 
         /**
          * Parses a string representation of a property into an actual Property instance. Not all properties that exist are

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java

@@ -18,6 +18,7 @@ import com.maxmind.geoip2.model.CityResponse;
 import com.maxmind.geoip2.model.CountryResponse;
 import com.maxmind.geoip2.model.DomainResponse;
 import com.maxmind.geoip2.model.EnterpriseResponse;
+import com.maxmind.geoip2.model.IspResponse;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -190,6 +191,12 @@ class DatabaseReaderLazyLoader implements GeoIpDatabase, Closeable {
         return getResponse(ipAddress, DatabaseReader::tryEnterprise);
     }
 
+    @Nullable
+    @Override
+    public IspResponse getIsp(InetAddress ipAddress) {
+        return getResponse(ipAddress, DatabaseReader::tryIsp);
+    }
+
     boolean preLookup() {
         return currentUsages.updateAndGet(current -> current < 0 ? current : current + 1) > 0;
     }

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java

@@ -14,6 +14,7 @@ import com.maxmind.geoip2.model.CityResponse;
 import com.maxmind.geoip2.model.CountryResponse;
 import com.maxmind.geoip2.model.DomainResponse;
 import com.maxmind.geoip2.model.EnterpriseResponse;
+import com.maxmind.geoip2.model.IspResponse;
 
 import org.elasticsearch.core.Nullable;
 
@@ -65,6 +66,9 @@ public interface GeoIpDatabase {
     @Nullable
     EnterpriseResponse getEnterprise(InetAddress ipAddress);
 
+    @Nullable
+    IspResponse getIsp(InetAddress ipAddress);
+
     /**
      * Releases the current database object. Called after processing a single document. Databases should be closed or returned to a
      * resource pool. No further interactions should be expected.

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java

@@ -15,6 +15,7 @@ import com.maxmind.geoip2.model.CityResponse;
 import com.maxmind.geoip2.model.CountryResponse;
 import com.maxmind.geoip2.model.DomainResponse;
 import com.maxmind.geoip2.model.EnterpriseResponse;
+import com.maxmind.geoip2.model.IspResponse;
 import com.maxmind.geoip2.record.City;
 import com.maxmind.geoip2.record.Continent;
 import com.maxmind.geoip2.record.Country;
@@ -178,6 +179,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
             case AnonymousIp -> retrieveAnonymousIpGeoData(geoIpDatabase, ipAddress);
             case Domain -> retrieveDomainGeoData(geoIpDatabase, ipAddress);
             case Enterprise -> retrieveEnterpriseGeoData(geoIpDatabase, ipAddress);
+            case Isp -> retrieveIspGeoData(geoIpDatabase, ipAddress);
         };
     }
 
@@ -424,6 +426,11 @@ public final class GeoIpProcessor extends AbstractProcessor {
         String organization_name = response.getTraits().getAutonomousSystemOrganization();
         Network network = response.getTraits().getNetwork();
 
+        String isp = response.getTraits().getIsp();
+        String ispOrganization = response.getTraits().getOrganization();
+        String mobileCountryCode = response.getTraits().getMobileCountryCode();
+        String mobileNetworkCode = response.getTraits().getMobileNetworkCode();
+
         boolean isHostingProvider = response.getTraits().isHostingProvider();
         boolean isTorExitNode = response.getTraits().isTorExitNode();
         boolean isAnonymousVpn = response.getTraits().isAnonymousVpn();
@@ -531,6 +538,84 @@ public final class GeoIpProcessor extends AbstractProcessor {
                         geoData.put("domain", domain);
                     }
                 }
+                case ISP -> {
+                    if (isp != null) {
+                        geoData.put("isp", isp);
+                    }
+                }
+                case ISP_ORGANIZATION_NAME -> {
+                    if (ispOrganization != null) {
+                        geoData.put("isp_organization", ispOrganization);
+                    }
+                }
+                case MOBILE_COUNTRY_CODE -> {
+                    if (mobileCountryCode != null) {
+                        geoData.put("mobile_country_code", mobileCountryCode);
+                    }
+                }
+                case MOBILE_NETWORK_CODE -> {
+                    if (mobileNetworkCode != null) {
+                        geoData.put("mobile_network_code", mobileNetworkCode);
+                    }
+                }
+            }
+        }
+        return geoData;
+    }
+
+    private Map<String, Object> retrieveIspGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
+        IspResponse response = geoIpDatabase.getIsp(ipAddress);
+        if (response == null) {
+            return Map.of();
+        }
+
+        String isp = response.getIsp();
+        String ispOrganization = response.getOrganization();
+        String mobileNetworkCode = response.getMobileNetworkCode();
+        String mobileCountryCode = response.getMobileCountryCode();
+        Long asn = response.getAutonomousSystemNumber();
+        String organization_name = response.getAutonomousSystemOrganization();
+        Network network = response.getNetwork();
+
+        Map<String, Object> geoData = new HashMap<>();
+        for (Property property : this.properties) {
+            switch (property) {
+                case IP -> geoData.put("ip", NetworkAddress.format(ipAddress));
+                case ASN -> {
+                    if (asn != null) {
+                        geoData.put("asn", asn);
+                    }
+                }
+                case ORGANIZATION_NAME -> {
+                    if (organization_name != null) {
+                        geoData.put("organization_name", organization_name);
+                    }
+                }
+                case NETWORK -> {
+                    if (network != null) {
+                        geoData.put("network", network.toString());
+                    }
+                }
+                case ISP -> {
+                    if (isp != null) {
+                        geoData.put("isp", isp);
+                    }
+                }
+                case ISP_ORGANIZATION_NAME -> {
+                    if (ispOrganization != null) {
+                        geoData.put("isp_organization", ispOrganization);
+                    }
+                }
+                case MOBILE_COUNTRY_CODE -> {
+                    if (mobileCountryCode != null) {
+                        geoData.put("mobile_country_code", mobileCountryCode);
+                    }
+                }
+                case MOBILE_NETWORK_CODE -> {
+                    if (mobileNetworkCode != null) {
+                        geoData.put("mobile_network_code", mobileNetworkCode);
+                    }
+                }
             }
         }
         return geoData;

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java

@@ -387,7 +387,7 @@ public class GeoIpProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData.size(), equalTo(19));
+        assertThat(geoData.size(), equalTo(21));
         assertThat(geoData.get("ip"), equalTo(ip));
         assertThat(geoData.get("country_iso_code"), equalTo("US"));
         assertThat(geoData.get("country_name"), equalTo("United States"));
@@ -410,6 +410,42 @@ public class GeoIpProcessorTests extends ESTestCase {
         assertThat(geoData.get("public_proxy"), equalTo(false));
         assertThat(geoData.get("residential_proxy"), equalTo(false));
         assertThat(geoData.get("domain"), equalTo("frpt.net"));
+        assertThat(geoData.get("isp"), equalTo("Fairpoint Communications"));
+        assertThat(geoData.get("isp_organization"), equalTo("Fairpoint Communications"));
+    }
+
+    public void testIsp() throws Exception {
+        String ip = "149.101.100.1";
+        GeoIpProcessor processor = new GeoIpProcessor(
+            randomAlphaOfLength(10),
+            null,
+            "source_field",
+            loader("/GeoIP2-ISP-Test.mmdb"),
+            () -> true,
+            "target_field",
+            ALL_PROPERTIES,
+            false,
+            false,
+            "filename"
+        );
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field", ip);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        processor.execute(ingestDocument);
+
+        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
+        @SuppressWarnings("unchecked")
+        Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
+        assertThat(geoData.size(), equalTo(8));
+        assertThat(geoData.get("ip"), equalTo(ip));
+        assertThat(geoData.get("asn"), equalTo(6167L));
+        assertThat(geoData.get("organization_name"), equalTo("CELLCO-PART"));
+        assertThat(geoData.get("network"), equalTo("149.101.100.0/28"));
+        assertThat(geoData.get("isp"), equalTo("Verizon Wireless"));
+        assertThat(geoData.get("isp_organization"), equalTo("Verizon Wireless"));
+        assertThat(geoData.get("mobile_network_code"), equalTo("004"));
+        assertThat(geoData.get("mobile_country_code"), equalTo("310"));
     }
 
     public void testAddressIsNotInTheDatabase() throws Exception {

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java

@@ -220,7 +220,11 @@ public class MaxMindSupportTests extends ESTestCase {
         "traits.autonomousSystemOrganization",
         "traits.domain",
         "traits.hostingProvider",
+        "traits.isp",
+        "traits.mobileCountryCode",
+        "traits.mobileNetworkCode",
         "traits.network",
+        "traits.organization",
         "traits.publicProxy",
         "traits.residentialProxy",
         "traits.torExitNode"
@@ -273,17 +277,25 @@ public class MaxMindSupportTests extends ESTestCase {
         "traits.anycast",
         "traits.connectionType",
         "traits.ipAddress",
-        "traits.isp",
         "traits.legitimateProxy",
-        "traits.mobileCountryCode",
-        "traits.mobileNetworkCode",
-        "traits.organization",
         "traits.satelliteProvider",
         "traits.staticIpScore",
         "traits.userCount",
         "traits.userType"
     );
 
+    private static final Set<String> ISP_SUPPORTED_FIELDS = Set.of(
+        "autonomousSystemNumber",
+        "autonomousSystemOrganization",
+        "network",
+        "isp",
+        "mobileCountryCode",
+        "mobileNetworkCode",
+        "organization"
+    );
+
+    private static final Set<String> ISP_UNSUPPORTED_FIELDS = Set.of("ipAddress");
+
     private static final Map<Database, Set<String>> TYPE_TO_SUPPORTED_FIELDS_MAP = Map.of(
         Database.AnonymousIp,
         ANONYMOUS_IP_SUPPORTED_FIELDS,
@@ -296,7 +308,9 @@ public class MaxMindSupportTests extends ESTestCase {
         Database.Domain,
         DOMAIN_SUPPORTED_FIELDS,
         Database.Enterprise,
-        ENTERPRISE_SUPPORTED_FIELDS
+        ENTERPRISE_SUPPORTED_FIELDS,
+        Database.Isp,
+        ISP_SUPPORTED_FIELDS
     );
     private static final Map<Database, Set<String>> TYPE_TO_UNSUPPORTED_FIELDS_MAP = Map.of(
         Database.AnonymousIp,
@@ -310,7 +324,9 @@ public class MaxMindSupportTests extends ESTestCase {
         Database.Domain,
         DOMAIN_UNSUPPORTED_FIELDS,
         Database.Enterprise,
-        ENTERPRISE_UNSUPPORTED_FIELDS
+        ENTERPRISE_UNSUPPORTED_FIELDS,
+        Database.Isp,
+        ISP_UNSUPPORTED_FIELDS
     );
     private static final Map<Database, Class<? extends AbstractResponse>> TYPE_TO_MAX_MIND_CLASS = Map.of(
         Database.AnonymousIp,
@@ -324,12 +340,13 @@ public class MaxMindSupportTests extends ESTestCase {
         Database.Domain,
         DomainResponse.class,
         Database.Enterprise,
-        EnterpriseResponse.class
+        EnterpriseResponse.class,
+        Database.Isp,
+        IspResponse.class
     );
 
     private static final Set<Class<? extends AbstractResponse>> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of(
         ConnectionTypeResponse.class,
-        IspResponse.class,
         IpRiskResponse.class
     );