Update GeoIP processor documentation (#71211)

This PR adds documentation for GeoIPv2 auto-update feature.
It also changes related settings names from geoip.downloader.* to ingest.geoip.downloader to have the same convention as current setting.

Relates to #68920

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: James Rodewig <40268737+jrodewig@users.noreply.github.com>

build.gradle

@@ -520,7 +520,7 @@ subprojects {
 subprojects { Project subproj ->
   plugins.withType(TestClustersPlugin).whenPluginAdded {
     testClusters.all {
-      systemProperty "geoip.downloader.enabled.default", "false"
+      systemProperty "ingest.geoip.downloader.enabled.default", "false"
     }
   }
 }

distribution/docker/docker-compose.yml

@@ -16,7 +16,7 @@ services:
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
        - node.store.allow_mmap=false
-       - geoip.downloader.enabled=false
+       - ingest.geoip.downloader.enabled=false
        - xpack.security.enabled=true
        - xpack.security.transport.ssl.enabled=true
        - xpack.security.http.ssl.enabled=true
@@ -69,7 +69,7 @@ services:
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
        - node.store.allow_mmap=false
-       - geoip.downloader.enabled=false
+       - ingest.geoip.downloader.enabled=false
        - xpack.security.enabled=true
        - xpack.security.transport.ssl.enabled=true
        - xpack.security.http.ssl.enabled=true

docs/build.gradle

@@ -62,6 +62,8 @@ testClusters.matching { it.name == "integTest"}.configureEach {
   if (singleNode().testDistribution == DEFAULT) {
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'indices.lifecycle.history_index_enabled', 'false'
+    setting 'ingest.geoip.downloader.enabled', 'false'
+    systemProperty 'es.geoip_v2_feature_flag_enabled', 'true'
     systemProperty 'es.shutdown_feature_flag_enabled', 'true'
     keystorePassword 'keystore-password'
   }

docs/reference/ingest/apis/geoip-stats-api.asciidoc

@@ -0,0 +1,93 @@
+[[geoip-stats-api]]
+=== GeoIP stats API
+++++
+<titleabbrev>GeoIP stats</titleabbrev>
+++++
+
+Gets download statistics for GeoIP2 databases used with the
+<<geoip-processor,`geoip` processor>>.
+
+[source,console]
+----
+GET _ingest/geoip/stats
+----
+
+[[geoip-stats-api-request]]
+==== {api-request-title}
+
+`GET _ingest/geoip/stats`
+
+[[geoip-stats-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the `monitor` or
+`manage` <<privileges-list-cluster,cluster privilege>> to use this API.
+
+* If <<ingest-geoip-downloader-enabled,`ingest.geoip.downloader.enabled`>> is
+disabled, this API returns zero values and an empty `nodes` object.
+
+[role="child_attributes"]
+[[geoip-stats-api-response-body]]
+==== {api-response-body-title}
+
+`stats`::
+(object)
+Download statistics for all GeoIP2 databases.
++
+.Properties of `stats`
+[%collapsible%open]
+====
+`successful_downloads`::
+(integer)
+Total number of successful database downloads.
+
+`failed_downloads`::
+(integer)
+Total number of failed database downloads.
+
+`total_download_time`::
+(integer)
+Total milliseconds spent downloading databases.
+
+`database_count`::
+(integer)
+Current number of databases available for use.
+
+`skipped_updates`::
+(integer)
+Total number of database updates skipped.
+====
+
+`nodes`::
+(object)
+Downloaded GeoIP2 databases for each node.
++
+.Properties of `nodes`
+[%collapsible%open]
+====
+`<node_id>`::
+(object)
+Downloaded databases for the node. The field key is the node ID.
++
+.Properties of `<node_id>`
+[%collapsible%open]
+=====
+`databases`::
+(array of objects)
+Downloaded databases for the node.
++
+.Properties of `databases` objects
+[%collapsible%open]
+======
+`name`::
+(string)
+Name of the database.
+======
+
+`files_in_temp`::
+(array of strings)
+Downloaded database files, including related license files. {es} stores these
+files in the node's <<es-tmpdir,temporary directory>>:
+`$ES_TMPDIR/geoip-databases/<node_id>`.
+=====
+====

docs/reference/ingest/apis/index.asciidoc

@@ -1,15 +1,29 @@
 [[ingest-apis]]
 == Ingest APIs
 
-The following ingest APIs are available for managing pipelines:
+Use ingest APIs to manage tasks and resources related to <<ingest,ingest
+pipelines>> and processors.
 
-* <<put-pipeline-api>> to add or update a pipeline
-* <<get-pipeline-api>> to return a specific pipeline
+[[ingest-pipeline-apis]]
+=== Ingest pipeline APIs
+
+Use the following APIs to create, manage, and test ingest pipelines:
+
+* <<put-pipeline-api>> to create or update a pipeline
+* <<get-pipeline-api>> to retrieve a pipeline configuration
 * <<delete-pipeline-api>> to delete a pipeline
-* <<simulate-pipeline-api>> to simulate a call to a pipeline
+* <<simulate-pipeline-api>> to test a pipeline
+
+[[ingest-stat-apis]]
+=== Stat APIs
 
+Use the following APIs to get statistics about ingest processing:
+
+* <<geoip-stats-api>> to get download statistics for GeoIP2 databases used with
+the <<geoip-processor,`geoip` processor>>.
 
 include::put-pipeline.asciidoc[]
-include::get-pipeline.asciidoc[]
 include::delete-pipeline.asciidoc[]
+include::get-pipeline.asciidoc[]
+include::geoip-stats-api.asciidoc[]
 include::simulate-pipeline.asciidoc[]

docs/reference/ingest/processors/geoip.asciidoc

@@ -4,21 +4,20 @@
 <titleabbrev>GeoIP</titleabbrev>
 ++++
 
-The `geoip` processor adds information about the geographical location of IP addresses, based on data from the Maxmind databases.
-This processor adds this information by default under the `geoip` field. The `geoip` processor can resolve both IPv4 and
-IPv6 addresses.
-
-The `ingest-geoip` module ships by default with the GeoLite2 City, GeoLite2 Country and GeoLite2 ASN GeoIP2 databases from Maxmind made available
-under the CCA-ShareAlike 4.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
-
-The `geoip` processor can run with other city, country and ASN GeoIP2 databases
-from Maxmind. The database files must be copied into the `ingest-geoip` config
-directory located at `$ES_CONFIG/ingest-geoip`. Custom database files must be
-stored uncompressed and the extension must be `-City.mmdb`, `-Country.mmdb`, or
-`-ASN.mmdb` to indicate the type of the database. These database files can not
-have the same filename as any of the built-in database names. The
-`database_file` processor option is used to specify the filename of the custom
-database to use for the processor.
+The `geoip` processor adds information about the geographical location of an
+IPv4 or IPv6 address.
+
+[[geoip-automatic-updates]]
+By default, the processor uses the GeoLite2 City, GeoLite2 Country, and GeoLite2
+ASN GeoIP2 databases from
+http://dev.maxmind.com/geoip/geoip2/geolite2/[MaxMind], shared under the
+CCA-ShareAlike 4.0 license. {es} automatically downloads updates for
+these databases from the Elastic GeoIP endpoint:
+https://geoip.elastic.co/v1/database. To get download statistics for these
+updates, use the <<geoip-stats-api,GeoIP stats API>>.
+
+If your cluster can't connect to the Elastic GeoIP endpoint or you want to
+manage your own updates, see <<manage-geoip-database-updates>>.
 
 [[using-ingest-geoip]]
 ==== Using the `geoip` Processor in a Pipeline
@@ -29,7 +28,7 @@ database to use for the processor.
 |======
 | Name                   | Required  | Default                                                                            | Description
 | `field`                | yes       | -                                                                                  | The field to get the ip address from for the geographical lookup.
-| `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the Maxmind database.
+| `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the MaxMind database.
 | `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename referring to a database the module ships with (GeoLite2-City.mmdb, GeoLite2-Country.mmdb, or GeoLite2-ASN.mmdb) or a custom database in the `ingest-geoip` config directory.
 | `properties`           | no        | [`continent_name`, `country_iso_code`, `country_name`, `region_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
 | `ignore_missing`       | no        | `false`                                                                            | If `true` and `field` does not exist, the processor quietly exits without modifying the document
@@ -300,6 +299,79 @@ GET /my_ip_locations/_search
 // TESTRESPONSE[s/"took" : 3/"took" : $body.took/]
 ////
 
+[[manage-geoip-database-updates]]
+==== Manage your own GeoIP2 database updates
+
+If you can't <<geoip-automatic-updates,automatically update>> your GeoIP2
+databases from the Elastic endpoint, you have a few other options:
+
+* <<use-proxy-geoip-endpoint,Use a proxy endpoint>>
+* <<use-custom-geoip-endpoint,Use a custom endpoint>>
+* <<manually-update-geoip-databases,Manually update your GeoIP2 databases>>
+
+[[use-proxy-geoip-endpoint]]
+**Use a proxy endpoint**
+
+If you can't connect directly to the Elastic GeoIP endpoint, consider setting up
+a secure proxy. You can then specify the proxy endpoint URL in the
+<<ingest-geoip-downloader-endpoint,`ingest.geoip.downloader.endpoint`>> setting
+of each node’s `elasticsearch.yml` file.
+
+[[use-custom-geoip-endpoint]]
+**Use a custom endpoint**
+
+You can create a service that mimics the Elastic GeoIP endpoint. You can then
+get automatic updates from this service.
+
+. Download your `.mmdb` database files from the
+http://dev.maxmind.com/geoip/geoip2/geolite2[MaxMind site].
+
+. Copy your database files to a single directory.
+
+. From your {es} directory, run:
++
+[source,sh]
+----
+./bin/elasticsearch-geoip -s my/source/dir [-t target/directory]
+----
+
+. Serve the static database files from your directory. For example, you can use
+Docker to serve the files from an nginx server:
++
+[source,sh]
+----
+docker run -v my/source/dir:/usr/share/nginx/html:ro nginx
+----
+
+. Specify the service's endpoint URL in the
+<<ingest-geoip-downloader-endpoint,`ingest.geoip.downloader.endpoint`>> setting
+of each node’s `elasticsearch.yml` file.
++
+By default, {es} checks the endpoint for updates every three days. To use
+another polling interval, use the <<cluster-update-settings,update cluster
+settings API>> to set
+<<ingest-geoip-downloader-poll-interval,`ingest.geoip.downloader.poll.interval`>>.
+
+[[manually-update-geoip-databases]]
+**Manually update your GeoIP2 databases**
+
+. Use the <<cluster-update-settings,update cluster settings API>> to set
+`ingest.geoip.downloader.enabled` to `false`. This disables automatic updates
+that may overwrite your database changes. This also deletes all downloaded
+databases.
+
+. Download your `.mmdb` database files from the
+http://dev.maxmind.com/geoip/geoip2/geolite2[MaxMind site].
++
+You can also use custom city, country, and ASN `.mmdb` files. These files must
+be uncompressed and use the respective `-City.mmdb`, `-Country.mmdb`, or
+`-ASN.mmdb` extensions.
+
+. Copy the database files to `$ES_CONFIG/ingest-geoip`.
+
+. In your `geoip` processors, configure the `database_file` parameter to use a
+custom database file.
+
 [[ingest-geoip-settings]]
 ===== Node Settings
 
@@ -310,3 +382,28 @@ The `geoip` processor supports the following setting:
     The maximum number of results that should be cached. Defaults to `1000`.
 
 Note that these settings are node settings and apply to all `geoip` processors, i.e. there is one cache for all defined `geoip` processors.
+
+[[geoip-cluster-settings]]
+===== Cluster settings
+
+[[ingest-geoip-downloader-enabled]]
+`ingest.geoip.downloader.enabled`::
+(<<dynamic-cluster-setting,Dynamic>>, Boolean)
+If `true`, {es} automatically downloads and manages updates for GeoIP2 databases
+from the `ingest.geoip.downloader.endpoint`. If `false`, {es} does not download
+updates and deletes all downloaded databases. Defaults to `true`.
+
+[[ingest-geoip-downloader-endpoint]]
+`ingest.geoip.downloader.endpoint`::
+(<<static-cluster-setting,Static>>, string)
+Endpoint URL used to download updates for GeoIP2 databases. Defaults to
+`https://geoip.elastic.co/v1/database`. {es} stores downloaded database files in
+each node's <<es-tmpdir,temporary directory>> at
+`$ES_TMPDIR/geoip-databases/<node_id>`.
+
+[[ingest-geoip-downloader-poll-interval]]
+`ingest.geoip.downloader.poll.interval`::
+(<<dynamic-cluster-setting,Dynamic>>, <<time-units,time value>>)
+How often {es} checks for GeoIP2 database updates at the
+`ingest.geoip.downloader.endpoint`. Must be greater than `1d` (one day). Defaults
+to `3d` (three days).

docs/reference/redirects.asciidoc

@@ -1513,8 +1513,3 @@ See <<put-enrich-policy-api>>.
 === Rollup API
 
 See <<rollup-apis>>.
-
-[role="exclude",id="geoip-stats-api"]
-=== GeoIP stats API
-
-coming::[7.x]

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java

@@ -58,9 +58,9 @@ public class GeoIpDownloader extends AllocatedPersistentTask {
 
     private static final Logger logger = LogManager.getLogger(GeoIpDownloader.class);
 
-    public static final Setting<TimeValue> POLL_INTERVAL_SETTING = Setting.timeSetting("geoip.downloader.poll.interval",
+    public static final Setting<TimeValue> POLL_INTERVAL_SETTING = Setting.timeSetting("ingest.geoip.downloader.poll.interval",
         TimeValue.timeValueDays(3), TimeValue.timeValueDays(1), Property.Dynamic, Property.NodeScope);
-    public static final Setting<String> ENDPOINT_SETTING = Setting.simpleString("geoip.downloader.endpoint",
+    public static final Setting<String> ENDPOINT_SETTING = Setting.simpleString("ingest.geoip.downloader.endpoint",
         "https://geoip.elastic.co/v1/database", Property.NodeScope);
 
     public static final String GEOIP_DOWNLOADER = "geoip-downloader";

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java

@@ -33,12 +33,12 @@ import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER;
 
 /**
  * Persistent task executor that is responsible for starting {@link GeoIpDownloader} after task is allocated by master node.
- * Also bootstraps GeoIP download task on clean cluster and handles changes to the 'geoip.downloader.enabled' setting
+ * Also bootstraps GeoIP download task on clean cluster and handles changes to the 'ingest.geoip.downloader.enabled' setting
  */
 public final class GeoIpDownloaderTaskExecutor extends PersistentTasksExecutor<GeoIpTaskParams> implements ClusterStateListener {
 
-    private static final boolean ENABLED_DEFAULT = "false".equals(System.getProperty("geoip.downloader.enabled.default")) == false;
-    public static final Setting<Boolean> ENABLED_SETTING = Setting.boolSetting("geoip.downloader.enabled", ENABLED_DEFAULT,
+    private static final boolean ENABLED_DEFAULT = "false".equals(System.getProperty("ingest.geoip.downloader.enabled.default")) == false;
+    public static final Setting<Boolean> ENABLED_SETTING = Setting.boolSetting("ingest.geoip.downloader.enabled", ENABLED_DEFAULT,
         Setting.Property.Dynamic, Setting.Property.NodeScope);
 
     private static final Logger logger = LogManager.getLogger(GeoIpDownloader.class);

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -86,7 +86,7 @@ public class DockerTests extends PackagingTestCase {
 
     @Before
     public void setupTest() throws IOException {
-        installation = runContainer(distribution(), builder().envVars(Map.of("geoip.downloader.enabled", "false")));
+        installation = runContainer(distribution(), builder().envVars(Map.of("ingest.geoip.downloader.enabled", "false")));
         tempDir = createTempDir(DockerTests.class.getSimpleName());
     }
 

qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java

@@ -95,7 +95,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
     public void test12InstallDockerDistribution() throws Exception {
         assumeTrue(distribution().isDocker());
 
-        installation = Docker.runContainer(distribution(), builder().envVars(Map.of("geoip.downloader.enabled", "false")));
+        installation = Docker.runContainer(distribution(), builder().envVars(Map.of("ingest.geoip.downloader.enabled", "false")));
 
         try {
             waitForPathToExist(installation.config("elasticsearch.keystore"));
@@ -273,7 +273,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
 
         // restart ES with password and mounted keystore
         Map<Path, Path> volumes = Map.of(localKeystoreFile, dockerKeystore);
-        Map<String, String> envVars = Map.of("KEYSTORE_PASSWORD", password, "geoip.downloader.enabled", "false");
+        Map<String, String> envVars = Map.of("KEYSTORE_PASSWORD", password, "ingest.geoip.downloader.enabled", "false");
         runContainer(distribution(), builder().volumes(volumes).envVars(envVars));
         waitForElasticsearch(installation);
         ServerUtils.runElasticsearchTests();
@@ -304,7 +304,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
             Map<String, String> envVars = Map.of(
                 "KEYSTORE_PASSWORD_FILE",
                 "/run/secrets/" + passwordFilename,
-                "geoip.downloader.enabled",
+                "ingest.geoip.downloader.enabled",
                 "false"
             );
 

qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -252,7 +252,7 @@ public class ServerUtils {
     }
 
     public static void disableGeoIpDownloader(Installation installation) throws IOException {
-        List<String> yaml = Collections.singletonList("geoip.downloader.enabled: false");
+        List<String> yaml = Collections.singletonList("ingest.geoip.downloader.enabled: false");
         Files.write(installation.config("elasticsearch.yml"), yaml, CREATE, APPEND);
     }
 }

qa/remote-clusters/docker-compose.yml

@@ -16,7 +16,7 @@ services:
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
        - node.store.allow_mmap=false
-       - geoip.downloader.enabled=false
+       - ingest.geoip.downloader.enabled=false
        - xpack.security.enabled=true
        - xpack.security.transport.ssl.enabled=true
        - xpack.security.http.ssl.enabled=true
@@ -70,7 +70,7 @@ services:
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
        - node.store.allow_mmap=false
-       - geoip.downloader.enabled=false
+       - ingest.geoip.downloader.enabled=false
        - xpack.security.enabled=true
        - xpack.security.transport.ssl.enabled=true
        - xpack.security.http.ssl.enabled=true

x-pack/test/idp-fixture/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - cluster.routing.allocation.disk.watermark.high=1b
       - cluster.routing.allocation.disk.watermark.flood_stage=1b
       - node.store.allow_mmap=false
-      - geoip.downloader.enabled=false
+      - ingest.geoip.downloader.enabled=false
       - xpack.license.self_generated.type=trial
       - xpack.security.enabled=true
       - xpack.security.http.ssl.enabled=true