New HTTP info (`/_info/http`) endpoint (#96198)

Adding a new endpoint under `_info/http`. This endpoint summarises the HTTP info of all the nodes into one big response, at cluster level. Compared with `_nodes/stats`, it lacks the nodes dimension.

docs/changelog/96198.yaml

@@ -0,0 +1,6 @@
+pr: 96198
+summary: New HTTP info endpoint
+area: Stats
+type: feature
+issues:
+ - 95391

docs/reference/cluster.asciidoc

@@ -104,6 +104,8 @@ include::cluster/nodes-reload-secure-settings.asciidoc[]
 
 include::cluster/nodes-stats.asciidoc[]
 
+include::cluster/info-http.asciidoc[]
+
 include::cluster/pending.asciidoc[]
 
 include::cluster/remote-info.asciidoc[]

docs/reference/cluster/info-http.asciidoc

@@ -0,0 +1,121 @@
+[[cluster-info-http]]
+=== Cluster Info HTTP API
+++++
+<titleabbrev>Cluster HTTP Info</titleabbrev>
+++++
+
+Returns cluster HTTP information.
+
+[[cluster-info-http-api-request]]
+==== {api-request-title}
+
+`GET /_info/http` +
+
+[[cluster-info-http-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the `monitor` or
+`manage` <<privileges-list-cluster,cluster privilege>> to use this API.
+
+
+[[cluster-info-http-api-desc]]
+==== {api-description-title}
+
+You can use the Cluster Info HTTP API to retrieve http information in a cluster.
+
+[role="child_attributes"]
+[[cluster-info-http-api-response-body]]
+==== {api-response-body-title}
+
+`cluster_name`::
+(string)
+Name of the cluster. Based on the <<cluster-name>> setting.
+
+
+[[cluster-info-http-api-response-body-http]]
+`http`::
+(object)
+Contains http statistics for the cluster.
++
+.Properties of `http`
+[%collapsible%open]
+======
+`current_open`::
+(integer)
+Current number of open HTTP connections for the cluster.
+
+`total_opened`::
+(integer)
+Total number of HTTP connections opened for the cluster.
+
+`clients`::
+(array of objects)
+Information on current and recently-closed HTTP client connections.
+Clients that have been closed longer than the <<http-settings,http.client_stats.closed_channels.max_age>>
+setting will not be represented here.
++
+.Properties of `clients`
+[%collapsible%open]
+=======
+`id`::
+(integer)
+Unique ID for the HTTP client.
+
+`agent`::
+(string)
+Reported agent for the HTTP client. If unavailable, this property is not
+included in the response.
+
+`local_address`::
+(string)
+Local address for the HTTP connection.
+
+`remote_address`::
+(string)
+Remote address for the HTTP connection.
+
+`last_uri`::
+(string)
+The URI of the client's most recent request.
+
+`x_forwarded_for`::
+(string)
+Value from the client's `x-forwarded-for` HTTP header. If unavailable, this
+property is not included in the response.
+
+`x_opaque_id`::
+(string)
+Value from the client's `x-opaque-id` HTTP header. If unavailable, this property
+is not included in the response.
+
+`opened_time_millis`::
+(integer)
+Time at which the client opened the connection.
+
+`closed_time_millis`::
+(integer)
+Time at which the client closed the connection if the connection is closed.
+
+`last_request_time_millis`::
+(integer)
+Time of the most recent request from this client.
+
+`request_count`::
+(integer)
+Number of requests from this client.
+
+`request_size_bytes`::
+(integer)
+Cumulative size in bytes of all requests from this client.
+=======
+======
+
+
+[[cluster-info-http-api-example]]
+==== {api-examples-title}
+
+[source,console]
+----
+# returns the http info of the cluster
+GET /_info/http
+----

docs/reference/cluster/nodes-stats.asciidoc

@@ -2114,11 +2114,11 @@ included in the response.
 
 `local_address`::
 (string)
-Local address for the HTTP client.
+Local address for the HTTP connection.
 
 `remote_address`::
 (string)
-Remote address for the HTTP client.
+Remote address for the HTTP connection.
 
 `last_uri`::
 (string)

rest-api-spec/src/main/resources/rest-api-spec/api/info.http.json

@@ -0,0 +1,22 @@
+{
+  "info.http": {
+      "documentation": {
+          "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-info-http.html",
+          "description": "Returns statistical information about http status in the cluster."
+      },
+      "stability": "stable",
+      "visibility": "public",
+      "headers": {
+          "accept": ["application/json"]
+      },
+      "url": {
+          "paths": [
+              {
+                  "path": "/_info/http",
+                  "methods": ["GET"]
+              }
+          ]
+      },
+      "params": {}
+  }
+}

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/info.http/10_info_http.yml

@@ -0,0 +1,24 @@
+---
+"HTTP Stats":
+  - skip:
+      version: " - 8.8.99"
+      reason: "/_info/http only available from v8.9"
+
+  - do:
+      info.http: {}
+
+  - is_true: cluster_name
+  - gte: { http.current_open: 0 }
+  - gte: { http.total_opened: 1 }
+  - is_true: http.clients
+  - gte: { http.clients.0.id: 1 }
+  - match: { http.clients.0.agent: "/.*/" }
+  - match: { http.clients.0.local_address: "/.*/" }
+  - match: { http.clients.0.remote_address: "/.*/" }
+  - is_true:  http.clients.0.last_uri
+  - gte: { http.clients.0.opened_time_millis: 1684328268000 } # 2023-05-17
+  - gte: { http.clients.0.last_request_time_millis: 1684328268000 }
+  - gte: { http.clients.0.request_count: 1 }
+  - gte: { http.clients.0.request_size_bytes: 0 }
+    # values for clients.0.closed_time_millis, clients.0.x_forwarded_for, and clients.0.x_opaque_id are often
+    # null and cannot be tested here

server/src/main/java/org/elasticsearch/action/ActionModule.java

@@ -420,6 +420,7 @@ import org.elasticsearch.rest.action.document.RestMultiGetAction;
 import org.elasticsearch.rest.action.document.RestMultiTermVectorsAction;
 import org.elasticsearch.rest.action.document.RestTermVectorsAction;
 import org.elasticsearch.rest.action.document.RestUpdateAction;
+import org.elasticsearch.rest.action.info.RestHttpInfoAction;
 import org.elasticsearch.rest.action.ingest.RestDeletePipelineAction;
 import org.elasticsearch.rest.action.ingest.RestGetPipelineAction;
 import org.elasticsearch.rest.action.ingest.RestPutPipelineAction;
@@ -935,6 +936,7 @@ public class ActionModule extends AbstractModule {
         registerHandler.accept(new RestShardsAction());
         registerHandler.accept(new RestMasterAction());
         registerHandler.accept(new RestNodesAction());
+        registerHandler.accept(new RestHttpInfoAction());
         registerHandler.accept(new RestTasksAction(nodesInCluster));
         registerHandler.accept(new RestIndicesAction());
         registerHandler.accept(new RestSegmentsAction());

server/src/main/java/org/elasticsearch/http/HttpStats.java

@@ -20,9 +20,12 @@ import org.elasticsearch.xcontent.XContentBuilder;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
+import java.util.stream.Stream;
 
 public record HttpStats(long serverOpen, long totalOpen, List<ClientStats> clientStats) implements Writeable, ChunkedToXContent {
 
+    public static final HttpStats IDENTITY = new HttpStats(0, 0, List.of());
+
     public HttpStats(long serverOpen, long totalOpened) {
         this(serverOpen, totalOpened, List.of());
     }
@@ -50,6 +53,14 @@ public record HttpStats(long serverOpen, long totalOpen, List<ClientStats> clien
         return this.clientStats;
     }
 
+    public static HttpStats merge(HttpStats first, HttpStats second) {
+        return new HttpStats(
+            first.serverOpen + second.serverOpen,
+            first.totalOpen + second.totalOpen,
+            Stream.concat(first.clientStats.stream(), second.clientStats.stream()).toList()
+        );
+    }
+
     static final class Fields {
         static final String HTTP = "http";
         static final String CURRENT_OPEN = "current_open";

server/src/main/java/org/elasticsearch/rest/action/info/AbstractInfoAction.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.rest.action.info;
+
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.ChunkedRestResponseBody;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestResponseListener;
+
+import java.io.IOException;
+
+import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
+
+public abstract class AbstractInfoAction extends BaseRestHandler {
+
+    public abstract NodesStatsRequest buildNodeStatsRequest();
+
+    public abstract ChunkedToXContent xContentChunks(NodesStatsResponse nodesStatsResponse);
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        return channel -> client.admin().cluster().nodesStats(buildNodeStatsRequest(), new RestResponseListener<>(channel) {
+            @Override
+            public RestResponse buildResponse(NodesStatsResponse nodesStatsResponse) throws Exception {
+                return new RestResponse(
+                    RestStatus.OK,
+                    ChunkedRestResponseBody.fromXContent(
+                        outerParams -> Iterators.concat(
+                            ChunkedToXContentHelper.startObject(),
+                            Iterators.single(
+                                (builder, params) -> builder.field("cluster_name", nodesStatsResponse.getClusterName().value())
+                            ),
+                            xContentChunks(nodesStatsResponse).toXContentChunked(outerParams),
+                            ChunkedToXContentHelper.endObject()
+                        ),
+                        EMPTY_PARAMS,
+                        channel
+                    )
+                );
+            }
+        });
+    }
+}

server/src/main/java/org/elasticsearch/rest/action/info/RestHttpInfoAction.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.rest.action.info;
+
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.http.HttpStats;
+import org.elasticsearch.rest.RestRequest;
+
+import java.util.List;
+
+public class RestHttpInfoAction extends AbstractInfoAction {
+
+    public static final NodesStatsRequest NODES_STATS_REQUEST = new NodesStatsRequest().clear()
+        .addMetric(NodesStatsRequest.Metric.HTTP.metricName());
+
+    @Override
+    public String getName() {
+        return "http_info_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(RestRequest.Method.GET, "/_info/http"));
+    }
+
+    @Override
+    public NodesStatsRequest buildNodeStatsRequest() {
+        return NODES_STATS_REQUEST;
+    }
+
+    @Override
+    public ChunkedToXContent xContentChunks(NodesStatsResponse nodesStatsResponse) {
+        return nodesStatsResponse.getNodes().stream().map(NodeStats::getHttp).reduce(HttpStats.IDENTITY, HttpStats::merge);
+    }
+}

server/src/test/java/org/elasticsearch/http/HttpStatsTests.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.http;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.hasSize;
+
+public class HttpStatsTests extends ESTestCase {
+
+    public void testMerge() {
+        var first = randomHttpStats();
+        var second = randomHttpStats();
+
+        var merged = HttpStats.merge(first, second);
+
+        assertEquals(merged.getServerOpen(), first.getServerOpen() + second.getServerOpen());
+        assertEquals(merged.getTotalOpen(), first.getTotalOpen() + second.getTotalOpen());
+        assertThat(merged.getClientStats(), hasSize(first.getClientStats().size() + second.getClientStats().size()));
+        assertEquals(merged.getClientStats(), Stream.concat(first.getClientStats().stream(), second.getClientStats().stream()).toList());
+    }
+
+    public static HttpStats randomHttpStats() {
+        return new HttpStats(
+            randomLongBetween(0, Long.MAX_VALUE),
+            randomLongBetween(0, Long.MAX_VALUE),
+            IntStream.range(1, randomIntBetween(2, 10)).mapToObj(HttpStatsTests::randomClients).toList()
+        );
+    }
+
+    public static HttpStats.ClientStats randomClients(int i) {
+        return new HttpStats.ClientStats(
+            randomInt(),
+            randomAlphaOfLength(100),
+            randomAlphaOfLength(100),
+            randomAlphaOfLength(100),
+            randomAlphaOfLength(100),
+            randomAlphaOfLength(100),
+            randomAlphaOfLength(100),
+            randomLong(),
+            randomLong(),
+            randomLong(),
+            randomLong(),
+            randomLong()
+        );
+    }
+}

server/src/test/java/org/elasticsearch/rest/action/info/RestHttpInfoActionTests.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.rest.action.info;
+
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.http.HttpStats;
+import org.elasticsearch.http.HttpStatsTests;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import static org.mockito.Mockito.mock;
+
+public class RestHttpInfoActionTests extends ESTestCase {
+
+    public void testXContentToChunks() {
+        var nodeStats = IntStream.range(1, randomIntBetween(2, 20)).mapToObj(this::randomNodeStatsWithOnlyHttpStats).toList();
+        var response = new NodesStatsResponse(new ClusterName("cluster-name"), nodeStats, List.of());
+
+        var httpStats = (HttpStats) new RestHttpInfoAction().xContentChunks(response);
+
+        assertEquals(
+            httpStats,
+            new HttpStats(
+                nodeStats.stream().map(NodeStats::getHttp).mapToLong(HttpStats::serverOpen).sum(),
+                nodeStats.stream().map(NodeStats::getHttp).mapToLong(HttpStats::totalOpen).sum(),
+                nodeStats.stream()
+                    .map(NodeStats::getHttp)
+                    .map(HttpStats::clientStats)
+                    .map(Collection::stream)
+                    .reduce(Stream.of(), Stream::concat)
+                    .toList()
+            )
+        );
+    }
+
+    private NodeStats randomNodeStatsWithOnlyHttpStats(int i) {
+        return new NodeStats(
+            mock(DiscoveryNode.class),
+            randomLong(),
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            HttpStatsTests.randomHttpStats(),
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+    }
+}