[Profiling] Add TopN Functions API (#106860)

With this commit we add a new API to the Universal Profiling plugin that
allows to gather a list of functions with the most observed samples
(TopN functions).

---------

Co-authored-by: Joseph Crail <joseph.crail@elastic.co>

docs/changelog/106860.yaml

@@ -0,0 +1,5 @@
+pr: 106860
+summary: "[Profiling] Add TopN Functions API"
+area: Application
+type: enhancement
+issues: []

rest-api-spec/src/main/resources/rest-api-spec/api/profiling.topn_functions.json

@@ -0,0 +1,28 @@
+{
+  "profiling.topn_functions":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/observability/current/universal-profiling.html",
+      "description":"Extracts a list of topN functions from Universal Profiling."
+    },
+    "stability":"stable",
+    "visibility":"private",
+    "headers":{
+      "accept": ["application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_profiling/topn/functions",
+          "methods":[
+            "POST"
+          ]
+        }
+      ]
+    },
+    "body":{
+      "description":"The filter conditions for stacktraces",
+      "required":true
+    }
+  }
+}

x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetFlameGraphActionIT.java

@@ -9,7 +9,21 @@ package org.elasticsearch.xpack.profiling;
 
 public class GetFlameGraphActionIT extends ProfilingTestCase {
     public void testGetStackTracesUnfiltered() throws Exception {
-        GetStackTracesRequest request = new GetStackTracesRequest(1000, 600.0d, 1.0d, 1.0d, null, null, null, null, null, null, null, null);
+        GetStackTracesRequest request = new GetStackTracesRequest(
+            1000,
+            600.0d,
+            1.0d,
+            1.0d,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null
+        );
         GetFlamegraphResponse response = client().execute(GetFlamegraphAction.INSTANCE, request).get();
         // only spot-check top level properties - detailed tests are done in unit tests
         assertEquals(994, response.getSize());

x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java

@@ -15,7 +15,65 @@ import java.util.List;
 
 public class GetStackTracesActionIT extends ProfilingTestCase {
     public void testGetStackTracesUnfiltered() throws Exception {
-        GetStackTracesRequest request = new GetStackTracesRequest(1000, 600.0d, 1.0d, 1.0d, null, null, null, null, null, null, null, null);
+        GetStackTracesRequest request = new GetStackTracesRequest(
+            1000,
+            600.0d,
+            1.0d,
+            1.0d,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+        request.setAdjustSampleCount(true);
+        GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
+        assertEquals(46, response.getTotalSamples());
+        assertEquals(1821, response.getTotalFrames());
+
+        assertNotNull(response.getStackTraceEvents());
+        assertEquals(3L, response.getStackTraceEvents().get("L7kj7UvlKbT-vN73el4faQ").count);
+
+        assertNotNull(response.getStackTraces());
+        // just do a high-level spot check. Decoding is tested in unit-tests
+        StackTrace stackTrace = response.getStackTraces().get("L7kj7UvlKbT-vN73el4faQ");
+        assertEquals(18, stackTrace.addressOrLines.length);
+        assertEquals(18, stackTrace.fileIds.length);
+        assertEquals(18, stackTrace.frameIds.length);
+        assertEquals(18, stackTrace.typeIds.length);
+        assertEquals(0.0000048475146d, stackTrace.annualCO2Tons, 0.0000000001d);
+        assertEquals(0.18834d, stackTrace.annualCostsUSD, 0.00001d);
+        // not determined by default
+        assertNull(stackTrace.subGroups);
+
+        assertNotNull(response.getStackFrames());
+        StackFrame stackFrame = response.getStackFrames().get("8NlMClggx8jaziUTJXlmWAAAAAAAAIYI");
+        assertEquals(List.of("start_thread"), stackFrame.functionName);
+
+        assertNotNull(response.getExecutables());
+        assertEquals("vmlinux", response.getExecutables().get("lHp5_WAgpLy2alrUVab6HA"));
+    }
+
+    public void testGetStackTracesGroupedByServiceName() throws Exception {
+        GetStackTracesRequest request = new GetStackTracesRequest(
+            1000,
+            600.0d,
+            1.0d,
+            1.0d,
+            null,
+            null,
+            null,
+            "service.name",
+            null,
+            null,
+            null,
+            null,
+            null
+        );
         request.setAdjustSampleCount(true);
         GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
         assertEquals(46, response.getTotalSamples());
@@ -33,6 +91,7 @@ public class GetStackTracesActionIT extends ProfilingTestCase {
         assertEquals(18, stackTrace.typeIds.length);
         assertEquals(0.0000048475146d, stackTrace.annualCO2Tons, 0.0000000001d);
         assertEquals(0.18834d, stackTrace.annualCostsUSD, 0.00001d);
+        assertEquals(Long.valueOf(2L), stackTrace.subGroups.get("basket"));
 
         assertNotNull(response.getStackFrames());
         StackFrame stackFrame = response.getStackFrames().get("8NlMClggx8jaziUTJXlmWAAAAAAAAIYI");
@@ -42,6 +101,28 @@ public class GetStackTracesActionIT extends ProfilingTestCase {
         assertEquals("vmlinux", response.getExecutables().get("lHp5_WAgpLy2alrUVab6HA"));
     }
 
+    public void testGetStackTracesGroupedByInvalidField() {
+        GetStackTracesRequest request = new GetStackTracesRequest(
+            1000,
+            600.0d,
+            1.0d,
+            1.0d,
+            null,
+            null,
+            null,
+            // only service.name is supported (note the trailing "s")
+            "service.names",
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+        request.setAdjustSampleCount(true);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, client().execute(GetStackTracesAction.INSTANCE, request));
+        assertEquals("Requested custom event aggregation field [service.names] but only [service.name] is supported.", e.getMessage());
+    }
+
     public void testGetStackTracesFromAPMWithMatchNoDownsampling() throws Exception {
         BoolQueryBuilder query = QueryBuilders.boolQuery();
         query.must().add(QueryBuilders.termQuery("transaction.name", "encodeSha1"));
@@ -56,6 +137,7 @@ public class GetStackTracesActionIT extends ProfilingTestCase {
             // also match an index that does not contain stacktrace ids to ensure it is ignored
             new String[] { "apm-test-*", "apm-legacy-test-*" },
             "transaction.profiler_stack_trace_ids",
+            "transaction.name",
             null,
             null,
             null,
@@ -79,6 +161,7 @@ public class GetStackTracesActionIT extends ProfilingTestCase {
         assertEquals(39, stackTrace.typeIds.length);
         assertTrue(stackTrace.annualCO2Tons > 0.0d);
         assertTrue(stackTrace.annualCostsUSD > 0.0d);
+        assertEquals(Long.valueOf(3L), stackTrace.subGroups.get("encodeSha1"));
 
         assertNotNull(response.getStackFrames());
         StackFrame stackFrame = response.getStackFrames().get("fhsEKXDuxJ-jIJrZpdRuSAAAAAAAAFtj");
@@ -103,6 +186,7 @@ public class GetStackTracesActionIT extends ProfilingTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         // ensures consistent results in the random sampler aggregation that is used internally
@@ -126,6 +210,8 @@ public class GetStackTracesActionIT extends ProfilingTestCase {
         assertEquals(39, stackTrace.typeIds.length);
         assertTrue(stackTrace.annualCO2Tons > 0.0d);
         assertTrue(stackTrace.annualCostsUSD > 0.0d);
+        // not determined by default
+        assertNull(stackTrace.subGroups);
 
         assertNotNull(response.getStackFrames());
         StackFrame stackFrame = response.getStackFrames().get("fhsEKXDuxJ-jIJrZpdRuSAAAAAAAAFtj");
@@ -150,6 +236,7 @@ public class GetStackTracesActionIT extends ProfilingTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
@@ -171,6 +258,7 @@ public class GetStackTracesActionIT extends ProfilingTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
@@ -192,6 +280,7 @@ public class GetStackTracesActionIT extends ProfilingTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();

x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetTopNFunctionsActionIT.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiling;
+
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+
+public class GetTopNFunctionsActionIT extends ProfilingTestCase {
+    public void testGetTopNFunctionsUnfiltered() throws Exception {
+        GetStackTracesRequest request = new GetStackTracesRequest(
+            1000,
+            600.0d,
+            1.0d,
+            1.0d,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+        request.setAdjustSampleCount(true);
+        GetTopNFunctionsResponse response = client().execute(GetTopNFunctionsAction.INSTANCE, request).get();
+        assertEquals(747, response.getTopN().size());
+    }
+
+    public void testGetTopNFunctionsGroupedByServiceName() throws Exception {
+        GetStackTracesRequest request = new GetStackTracesRequest(
+            1000,
+            600.0d,
+            1.0d,
+            1.0d,
+            null,
+            null,
+            null,
+            "service.name",
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+        request.setAdjustSampleCount(true);
+        request.setLimit(50);
+        GetTopNFunctionsResponse response = client().execute(GetTopNFunctionsAction.INSTANCE, request).get();
+        assertEquals(50, response.getTopN().size());
+    }
+
+    public void testGetTopNFunctionsFromAPM() throws Exception {
+        BoolQueryBuilder query = QueryBuilders.boolQuery();
+        query.must().add(QueryBuilders.termQuery("transaction.name", "encodeSha1"));
+        query.must().add(QueryBuilders.rangeQuery("@timestamp").lte("1698624000"));
+
+        GetStackTracesRequest request = new GetStackTracesRequest(
+            null,
+            1.0d,
+            1.0d,
+            1.0d,
+            query,
+            // also match an index that does not contain stacktrace ids to ensure it is ignored
+            new String[] { "apm-test-*", "apm-legacy-test-*" },
+            "transaction.profiler_stack_trace_ids",
+            "transaction.name",
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+        GetTopNFunctionsResponse response = client().execute(GetTopNFunctionsAction.INSTANCE, request).get();
+        assertEquals(45, response.getTopN().size());
+    }
+}

x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson

@@ -71,9 +71,9 @@
 {"create": {"_index": "profiling-events-all"}}
 {"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["XF9MchOwpePfa6_hYy-vZQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
 {"create": {"_index": "profiling-events-all"}}
-{"Stacktrace.count": [2], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
+{"Stacktrace.count": [2], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"], "service.name":  "basket"}
 {"create": {"_index": "profiling-events-all"}}
-{"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
+{"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"], "service.name":  "basket"}
 {"create": {"_index": "profiling-events-all"}}
 {"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["hRqQI2CBPiapzgFG9jrmDA"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["599103450330106"]}
 {"create": {"_index": "profiling-events-all"}}

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesRequest.java

@@ -38,8 +38,10 @@ import static org.elasticsearch.index.query.AbstractQueryBuilder.parseTopLevelQu
 public class GetStackTracesRequest extends ActionRequest implements IndicesRequest.Replaceable {
     public static final ParseField QUERY_FIELD = new ParseField("query");
     public static final ParseField SAMPLE_SIZE_FIELD = new ParseField("sample_size");
+    public static final ParseField LIMIT_FIELD = new ParseField("limit");
     public static final ParseField INDICES_FIELD = new ParseField("indices");
     public static final ParseField STACKTRACE_IDS_FIELD = new ParseField("stacktrace_ids_field");
+    public static final ParseField AGGREGATION_FIELD = new ParseField("aggregation_field");
     public static final ParseField REQUESTED_DURATION_FIELD = new ParseField("requested_duration");
     public static final ParseField AWS_COST_FACTOR_FIELD = new ParseField("aws_cost_factor");
     public static final ParseField AZURE_COST_FACTOR_FIELD = new ParseField("azure_cost_factor");
@@ -52,9 +54,11 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
 
     private QueryBuilder query;
     private int sampleSize;
+    private Integer limit;
     private String[] indices;
     private boolean userProvidedIndices;
     private String stackTraceIdsField;
+    private String aggregationField;
     private Double requestedDuration;
     private Double awsCostFactor;
     private Double azureCostFactor;
@@ -73,7 +77,7 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
     private Integer shardSeed;
 
     public GetStackTracesRequest() {
-        this(null, null, null, null, null, null, null, null, null, null, null, null);
+        this(null, null, null, null, null, null, null, null, null, null, null, null, null);
     }
 
     public GetStackTracesRequest(
@@ -84,6 +88,7 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
         QueryBuilder query,
         String[] indices,
         String stackTraceIdsField,
+        String aggregationField,
         Double customCO2PerKWH,
         Double customDatacenterPUE,
         Double customPerCoreWattX86,
@@ -98,6 +103,7 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
         this.indices = indices;
         this.userProvidedIndices = indices != null && indices.length > 0;
         this.stackTraceIdsField = stackTraceIdsField;
+        this.aggregationField = aggregationField;
         this.customCO2PerKWH = customCO2PerKWH;
         this.customDatacenterPUE = customDatacenterPUE;
         this.customPerCoreWattX86 = customPerCoreWattX86;
@@ -114,6 +120,14 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
         return sampleSize;
     }
 
+    public void setLimit(int limit) {
+        this.limit = limit;
+    }
+
+    public Integer getLimit() {
+        return limit;
+    }
+
     public Double getRequestedDuration() {
         return requestedDuration;
     }
@@ -162,6 +176,10 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
         return stackTraceIdsField;
     }
 
+    public String getAggregationField() {
+        return aggregationField;
+    }
+
     public boolean isAdjustSampleCount() {
         return Boolean.TRUE.equals(adjustSampleCount);
     }
@@ -194,8 +212,12 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
             } else if (token.isValue()) {
                 if (SAMPLE_SIZE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     this.sampleSize = parser.intValue();
+                } else if (LIMIT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    this.limit = parser.intValue();
                 } else if (STACKTRACE_IDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     this.stackTraceIdsField = parser.text();
+                } else if (AGGREGATION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    this.aggregationField = parser.text();
                 } else if (REQUESTED_DURATION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     this.requestedDuration = parser.doubleValue();
                 } else if (AWS_COST_FACTOR_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -277,7 +299,15 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
                 );
             }
         }
+        if (aggregationField != null && aggregationField.isBlank()) {
+            validationException = addValidationError(
+                "[" + AGGREGATION_FIELD.getPreferredName() + "] must be non-empty",
+                validationException
+            );
+        }
+
         validationException = requirePositive(SAMPLE_SIZE_FIELD, sampleSize, validationException);
+        validationException = requirePositive(LIMIT_FIELD, limit, validationException);
         validationException = requirePositive(REQUESTED_DURATION_FIELD, requestedDuration, validationException);
         validationException = requirePositive(AWS_COST_FACTOR_FIELD, awsCostFactor, validationException);
         validationException = requirePositive(AZURE_COST_FACTOR_FIELD, azureCostFactor, validationException);
@@ -307,7 +337,9 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
                 StringBuilder sb = new StringBuilder();
                 appendField(sb, "indices", indices);
                 appendField(sb, "stacktrace_ids_field", stackTraceIdsField);
+                appendField(sb, "aggregation_field", aggregationField);
                 appendField(sb, "sample_size", sampleSize);
+                appendField(sb, "limit", limit);
                 appendField(sb, "requested_duration", requestedDuration);
                 appendField(sb, "aws_cost_factor", awsCostFactor);
                 appendField(sb, "azure_cost_factor", azureCostFactor);

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesResponseBuilder.java

@@ -155,6 +155,9 @@ class GetStackTracesResponseBuilder {
                 if (event != null) {
                     StackTrace stackTrace = entry.getValue();
                     stackTrace.count = event.count;
+                    if (event.subGroups.isEmpty() == false) {
+                        stackTrace.subGroups = event.subGroups;
+                    }
                     stackTrace.annualCO2Tons = event.annualCO2Tons;
                     stackTrace.annualCostsUSD = event.annualCostsUSD;
                 }

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetTopNFunctionsAction.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.profiling;
+
+import org.elasticsearch.action.ActionType;
+
+public final class GetTopNFunctionsAction extends ActionType<GetTopNFunctionsResponse> {
+    public static final GetTopNFunctionsAction INSTANCE = new GetTopNFunctionsAction();
+    public static final String NAME = "indices:data/read/profiling/topn/functions";
+
+    private GetTopNFunctionsAction() {
+        super(NAME);
+    }
+}

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetTopNFunctionsResponse.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiling;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+public class GetTopNFunctionsResponse extends ActionResponse implements ToXContentObject {
+    private final long selfCount;
+    private final long totalCount;
+    private final List<TopNFunction> topNFunctions;
+
+    public GetTopNFunctionsResponse(long selfCount, long totalCount, List<TopNFunction> topNFunctions) {
+        this.selfCount = selfCount;
+        this.totalCount = totalCount;
+        this.topNFunctions = topNFunctions;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) {
+        TransportAction.localOnly();
+    }
+
+    public long getSelfCount() {
+        return selfCount;
+    }
+
+    public long getTotalCount() {
+        return totalCount;
+    }
+
+    public List<TopNFunction> getTopN() {
+        return topNFunctions;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("self_count", selfCount);
+        builder.field("total_count", totalCount);
+        builder.xContentList("topn", topNFunctions);
+        builder.endObject();
+        return builder;
+    }
+}

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingPlugin.java

@@ -134,6 +134,7 @@ public class ProfilingPlugin extends Plugin implements ActionPlugin {
         if (enabled) {
             handlers.add(new RestGetStackTracesAction());
             handlers.add(new RestGetFlamegraphAction());
+            handlers.add(new RestGetTopNFunctionsAction());
         }
         return Collections.unmodifiableList(handlers);
     }
@@ -168,6 +169,7 @@ public class ProfilingPlugin extends Plugin implements ActionPlugin {
         return List.of(
             new ActionHandler<>(GetStackTracesAction.INSTANCE, TransportGetStackTracesAction.class),
             new ActionHandler<>(GetFlamegraphAction.INSTANCE, TransportGetFlamegraphAction.class),
+            new ActionHandler<>(GetTopNFunctionsAction.INSTANCE, TransportGetTopNFunctionsAction.class),
             new ActionHandler<>(GetStatusAction.INSTANCE, TransportGetStatusAction.class),
             new ActionHandler<>(XPackUsageFeatureAction.UNIVERSAL_PROFILING, ProfilingUsageTransportAction.class),
             new ActionHandler<>(XPackInfoFeatureAction.UNIVERSAL_PROFILING, ProfilingInfoTransportAction.class)

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/RestGetTopNFunctionsAction.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.profiling;
+
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+@ServerlessScope(Scope.PUBLIC)
+public class RestGetTopNFunctionsAction extends BaseRestHandler {
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(POST, "/_profiling/topn/functions"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        GetStackTracesRequest getStackTracesRequest = new GetStackTracesRequest();
+        request.applyContentParser(getStackTracesRequest::parseXContent);
+        // enforce server-side adjustment of sample counts for top N functions
+        getStackTracesRequest.setAdjustSampleCount(true);
+
+        return channel -> {
+            RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel());
+            cancelClient.execute(GetTopNFunctionsAction.INSTANCE, getStackTracesRequest, new RestToXContentListener<>(channel));
+        };
+    }
+
+    @Override
+    public String getName() {
+        return "get_topn_functions_action";
+    }
+}

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/StackFrame.java

@@ -18,6 +18,7 @@ import java.util.Objects;
 import java.util.function.Consumer;
 
 final class StackFrame implements ToXContentObject {
+    static final StackFrame EMPTY_STACKFRAME = new StackFrame("", "", 0, 0);
     List<String> fileName;
     List<String> functionName;
     List<Integer> functionOffset;

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/StackTrace.java

@@ -26,7 +26,7 @@ final class StackTrace implements ToXContentObject {
     String[] fileIds;
     String[] frameIds;
     int[] typeIds;
-
+    Map<String, Long> subGroups;
     double annualCO2Tons;
     double annualCostsUSD;
     long count;
@@ -247,10 +247,10 @@ final class StackTrace implements ToXContentObject {
             && Arrays.equals(fileIds, that.fileIds)
             && Arrays.equals(frameIds, that.frameIds)
             && Arrays.equals(typeIds, that.typeIds);
-        // Don't compare metadata like annualized co2, annualized costs and count.
+        // Don't compare metadata like annualized co2, annualized costs, subGroups and count.
     }
 
-    // Don't hash metadata like annualized co2, annualized costs and count.
+    // Don't hash metadata like annualized co2, annualized costs, subGroups and count.
     @Override
     public int hashCode() {
         int result = Arrays.hashCode(addressOrLines);

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TopNFunction.java

@@ -0,0 +1,297 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiling;
+
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+final class TopNFunction implements Cloneable, ToXContentObject, Comparable<TopNFunction> {
+    private final String id;
+    private int rank;
+    private final int frameType;
+    private final boolean inline;
+    private final int addressOrLine;
+    private final String functionName;
+    private final String sourceFilename;
+    private final int sourceLine;
+    private final String exeFilename;
+    private long selfCount;
+    private long totalCount;
+    private double selfAnnualCO2Tons;
+    private double totalAnnualCO2Tons;
+    private double selfAnnualCostsUSD;
+    private double totalAnnualCostsUSD;
+    private final Map<String, Long> subGroups;
+
+    TopNFunction(
+        String id,
+        int frameType,
+        boolean inline,
+        int addressOrLine,
+        String functionName,
+        String sourceFilename,
+        int sourceLine,
+        String exeFilename
+    ) {
+        this(
+            id,
+            0,
+            frameType,
+            inline,
+            addressOrLine,
+            functionName,
+            sourceFilename,
+            sourceLine,
+            exeFilename,
+            0,
+            0,
+            0.0d,
+            0.0d,
+            0.0d,
+            0.0d,
+            new HashMap<>()
+        );
+    }
+
+    TopNFunction(
+        String id,
+        int rank,
+        int frameType,
+        boolean inline,
+        int addressOrLine,
+        String functionName,
+        String sourceFilename,
+        int sourceLine,
+        String exeFilename,
+        long selfCount,
+        long totalCount,
+        double selfAnnualCO2Tons,
+        double totalAnnualCO2Tons,
+        double selfAnnualCostsUSD,
+        double totalAnnualCostsUSD,
+        Map<String, Long> subGroups
+    ) {
+        this.id = id;
+        this.rank = rank;
+        this.frameType = frameType;
+        this.inline = inline;
+        this.addressOrLine = addressOrLine;
+        this.functionName = functionName;
+        this.sourceFilename = sourceFilename;
+        this.sourceLine = sourceLine;
+        this.exeFilename = exeFilename;
+        this.selfCount = selfCount;
+        this.totalCount = totalCount;
+        this.selfAnnualCO2Tons = selfAnnualCO2Tons;
+        this.totalAnnualCO2Tons = totalAnnualCO2Tons;
+        this.selfAnnualCostsUSD = selfAnnualCostsUSD;
+        this.totalAnnualCostsUSD = totalAnnualCostsUSD;
+        this.subGroups = subGroups;
+    }
+
+    public String getId() {
+        return this.id;
+    }
+
+    public void setRank(int rank) {
+        this.rank = rank;
+    }
+
+    public long getSelfCount() {
+        return selfCount;
+    }
+
+    public void addSelfCount(long selfCount) {
+        this.selfCount += selfCount;
+    }
+
+    public long getTotalCount() {
+        return totalCount;
+    }
+
+    public void addTotalCount(long totalCount) {
+        this.totalCount += totalCount;
+    }
+
+    public void addSelfAnnualCO2Tons(double co2Tons) {
+        this.selfAnnualCO2Tons += co2Tons;
+    }
+
+    public void addTotalAnnualCO2Tons(double co2Tons) {
+        this.totalAnnualCO2Tons += co2Tons;
+    }
+
+    public void addSelfAnnualCostsUSD(double costs) {
+        this.selfAnnualCostsUSD += costs;
+    }
+
+    public void addTotalAnnualCostsUSD(double costs) {
+        this.totalAnnualCostsUSD += costs;
+    }
+
+    public void addSubGroups(Map<String, Long> subGroups) {
+        for (Map.Entry<String, Long> subGroup : subGroups.entrySet()) {
+            long count = this.subGroups.getOrDefault(subGroup.getKey(), 0L);
+            this.subGroups.put(subGroup.getKey(), count + subGroup.getValue());
+        }
+    }
+
+    @Override
+    protected TopNFunction clone() {
+        return new TopNFunction(
+            id,
+            rank,
+            frameType,
+            inline,
+            addressOrLine,
+            functionName,
+            sourceFilename,
+            sourceLine,
+            exeFilename,
+            selfCount,
+            totalCount,
+            selfAnnualCO2Tons,
+            totalAnnualCO2Tons,
+            selfAnnualCostsUSD,
+            totalAnnualCostsUSD,
+            new HashMap<>(subGroups)
+        );
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("id", this.id);
+        builder.field("rank", this.rank);
+        builder.startObject("frame");
+        builder.field("frame_type", this.frameType);
+        builder.field("inline", this.inline);
+        builder.field("address_or_line", this.addressOrLine);
+        builder.field("function_name", this.functionName);
+        builder.field("file_name", this.sourceFilename);
+        builder.field("line_number", this.sourceLine);
+        builder.field("executable_file_name", this.exeFilename);
+        builder.endObject();
+        builder.field("sub_groups", subGroups);
+        builder.field("self_count", this.selfCount);
+        builder.field("total_count", this.totalCount);
+        builder.field("self_annual_co2_tons").rawValue(NumberUtils.doubleToString(selfAnnualCO2Tons));
+        builder.field("total_annual_co2_tons").rawValue(NumberUtils.doubleToString(totalAnnualCO2Tons));
+        builder.field("self_annual_costs_usd").rawValue(NumberUtils.doubleToString(selfAnnualCostsUSD));
+        builder.field("total_annual_costs_usd").rawValue(NumberUtils.doubleToString(totalAnnualCostsUSD));
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        TopNFunction that = (TopNFunction) o;
+        return Objects.equals(id, that.id)
+            && Objects.equals(rank, that.rank)
+            && Objects.equals(frameType, that.frameType)
+            && Objects.equals(inline, that.inline)
+            && Objects.equals(addressOrLine, that.addressOrLine)
+            && Objects.equals(functionName, that.functionName)
+            && Objects.equals(sourceFilename, that.sourceFilename)
+            && Objects.equals(sourceLine, that.sourceLine)
+            && Objects.equals(exeFilename, that.exeFilename)
+            && Objects.equals(selfCount, that.selfCount)
+            && Objects.equals(totalCount, that.totalCount)
+            && Objects.equals(selfAnnualCO2Tons, that.selfAnnualCO2Tons)
+            && Objects.equals(totalAnnualCO2Tons, that.totalAnnualCO2Tons)
+            && Objects.equals(selfAnnualCostsUSD, that.selfAnnualCostsUSD)
+            && Objects.equals(totalAnnualCostsUSD, that.totalAnnualCostsUSD)
+            && Objects.equals(subGroups, that.subGroups);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            id,
+            rank,
+            frameType,
+            inline,
+            addressOrLine,
+            functionName,
+            sourceFilename,
+            sourceLine,
+            exeFilename,
+            selfCount,
+            totalCount,
+            selfAnnualCO2Tons,
+            totalAnnualCO2Tons,
+            selfAnnualCostsUSD,
+            totalAnnualCostsUSD,
+            subGroups
+        );
+    }
+
+    @Override
+    public String toString() {
+        return "TopNFunction{"
+            + "id='"
+            + id
+            + '\''
+            + ", rank="
+            + rank
+            + ", frameType="
+            + frameType
+            + ", inline="
+            + inline
+            + ", addressOrLine="
+            + addressOrLine
+            + ", functionName='"
+            + functionName
+            + '\''
+            + ", sourceFilename='"
+            + sourceFilename
+            + '\''
+            + ", sourceLine="
+            + sourceLine
+            + ", exeFilename='"
+            + exeFilename
+            + '\''
+            + ", selfCount="
+            + selfCount
+            + ", totalCount="
+            + totalCount
+            + ", selfAnnualCO2Tons="
+            + selfAnnualCO2Tons
+            + ", totalAnnualCO2Tons="
+            + totalAnnualCO2Tons
+            + ", selfAnnualCostsUSD="
+            + selfAnnualCostsUSD
+            + ", totalAnnualCostsUSD="
+            + totalAnnualCostsUSD
+            + ", subGroups="
+            + subGroups
+            + '}';
+    }
+
+    @Override
+    public int compareTo(TopNFunction that) {
+        if (this.selfCount > that.selfCount) {
+            return 1;
+        }
+        if (this.selfCount < that.selfCount) {
+            return -1;
+        }
+        return this.id.compareTo(that.id);
+    }
+}

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TraceEvent.java

@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.profiling;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
 
 final class TraceEvent {
@@ -14,9 +16,10 @@ final class TraceEvent {
     double annualCO2Tons;
     double annualCostsUSD;
     long count;
+    final Map<String, Long> subGroups = new HashMap<>();
 
     TraceEvent(String stacktraceID) {
-        this.stacktraceID = stacktraceID;
+        this(stacktraceID, 0);
     }
 
     TraceEvent(String stacktraceID, long count) {
@@ -53,6 +56,8 @@ final class TraceEvent {
             + annualCostsUSD
             + ", count="
             + count
-            + "}";
+            + ", subGroups="
+            + subGroups
+            + '}';
     }
 }

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetFlamegraphAction.java

@@ -28,8 +28,6 @@ import java.util.TreeMap;
 
 public class TransportGetFlamegraphAction extends TransportAction<GetStackTracesRequest, GetFlamegraphResponse> {
     private static final Logger log = LogManager.getLogger(TransportGetFlamegraphAction.class);
-    private static final StackFrame EMPTY_STACKFRAME = new StackFrame("", "", 0, 0);
-
     private final NodeClient nodeClient;
     private final TransportService transportService;
 
@@ -97,7 +95,7 @@ public class TransportGetFlamegraphAction extends TransportAction<GetStackTraces
                 String fileId = stackTrace.fileIds[i];
                 int frameType = stackTrace.typeIds[i];
                 int addressOrLine = stackTrace.addressOrLines[i];
-                StackFrame stackFrame = response.getStackFrames().getOrDefault(frameId, EMPTY_STACKFRAME);
+                StackFrame stackFrame = response.getStackFrames().getOrDefault(frameId, StackFrame.EMPTY_STACKFRAME);
                 String executable = response.getExecutables().getOrDefault(fileId, "");
                 final boolean isLeafFrame = i == frameCount - 1;
 

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java

@@ -108,6 +108,12 @@ public class TransportGetStackTracesAction extends TransportAction<GetStackTrace
      */
     private static final int MAX_TRACE_EVENTS_RESULT_SIZE = 150_000;
 
+    /**
+     * Users may provide a custom field via the API that is used to sub-divide profiling events. This is useful in the context of TopN
+     * where we want to provide additional breakdown of where a certain function has been called (e.g. a certain service or transaction).
+     */
+    private static final String CUSTOM_EVENT_SUB_AGGREGATION_NAME = "custom_event_group";
+
     private final NodeClient nodeClient;
     private final ProfilingLicenseChecker licenseChecker;
     private final ClusterService clusterService;
@@ -241,11 +247,25 @@ public class TransportGetStackTracesAction extends TransportAction<GetStackTrace
         GetStackTracesResponseBuilder responseBuilder
     ) {
 
+        CountedTermsAggregationBuilder groupByStackTraceId = new CountedTermsAggregationBuilder("group_by").size(
+            MAX_TRACE_EVENTS_RESULT_SIZE
+        ).field(request.getStackTraceIdsField());
+        if (request.getAggregationField() != null) {
+            String aggregationField = request.getAggregationField();
+            log.trace("Grouping stacktrace events by [{}].", aggregationField);
+            // be strict about the accepted field names to avoid downstream errors or leaking unintended information
+            if (aggregationField.equals("transaction.name") == false) {
+                throw new IllegalArgumentException(
+                    "Requested custom event aggregation field [" + aggregationField + "] but only [transaction.name] is supported."
+                );
+            }
+            groupByStackTraceId.subAggregation(
+                new TermsAggregationBuilder(CUSTOM_EVENT_SUB_AGGREGATION_NAME).field(request.getAggregationField())
+            );
+        }
         RandomSamplerAggregationBuilder randomSampler = new RandomSamplerAggregationBuilder("sample").setSeed(request.hashCode())
             .setProbability(responseBuilder.getSamplingRate())
-            .subAggregation(
-                new CountedTermsAggregationBuilder("group_by").size(MAX_TRACE_EVENTS_RESULT_SIZE).field(request.getStackTraceIdsField())
-            );
+            .subAggregation(groupByStackTraceId);
         // shard seed is only set in tests and ensures consistent results
         if (request.getShardSeed() != null) {
             randomSampler.setShardSeed(request.getShardSeed());
@@ -283,6 +303,14 @@ public class TransportGetStackTracesAction extends TransportAction<GetStackTrace
                         stackTraceEvents.put(stackTraceID, event);
                     }
                     event.count += count;
+                    if (request.getAggregationField() != null) {
+                        Terms eventSubGroup = stacktraceBucket.getAggregations().get(CUSTOM_EVENT_SUB_AGGREGATION_NAME);
+                        for (Terms.Bucket b : eventSubGroup.getBuckets()) {
+                            String subGroupName = b.getKeyAsString();
+                            long subGroupCount = event.subGroups.getOrDefault(subGroupName, 0L);
+                            event.subGroups.put(subGroupName, subGroupCount + b.getDocCount());
+                        }
+                    }
                 }
                 responseBuilder.setTotalSamples(totalSamples);
                 responseBuilder.setHostEventCounts(hostEventCounts);
@@ -300,6 +328,25 @@ public class TransportGetStackTracesAction extends TransportAction<GetStackTrace
         EventsIndex eventsIndex
     ) {
         responseBuilder.setSamplingRate(eventsIndex.getSampleRate());
+        TermsAggregationBuilder groupByStackTraceId = new TermsAggregationBuilder("group_by")
+            // 'size' should be max 100k, but might be slightly more. Better be on the safe side.
+            .size(MAX_TRACE_EVENTS_RESULT_SIZE)
+            .field("Stacktrace.id")
+            // 'execution_hint: map' skips the slow building of ordinals that we don't need.
+            // Especially with high cardinality fields, this makes aggregations really slow.
+            .executionHint("map")
+            .subAggregation(new SumAggregationBuilder("count").field("Stacktrace.count"));
+        if (request.getAggregationField() != null) {
+            String aggregationField = request.getAggregationField();
+            log.trace("Grouping stacktrace events by [{}].", aggregationField);
+            // be strict about the accepted field names to avoid downstream errors or leaking unintended information
+            if (aggregationField.equals("service.name") == false) {
+                throw new IllegalArgumentException(
+                    "Requested custom event aggregation field [" + aggregationField + "] but only [service.name] is supported."
+                );
+            }
+            groupByStackTraceId.subAggregation(new TermsAggregationBuilder(CUSTOM_EVENT_SUB_AGGREGATION_NAME).field(aggregationField));
+        }
         client.prepareSearch(eventsIndex.getName())
             .setTrackTotalHits(false)
             .setSize(0)
@@ -320,16 +367,7 @@ public class TransportGetStackTracesAction extends TransportAction<GetStackTrace
                     // 'execution_hint: map' skips the slow building of ordinals that we don't need.
                     // Especially with high cardinality fields, this makes aggregations really slow.
                     .executionHint("map")
-                    .subAggregation(
-                        new TermsAggregationBuilder("group_by")
-                            // 'size' should be max 100k, but might be slightly more. Better be on the safe side.
-                            .size(MAX_TRACE_EVENTS_RESULT_SIZE)
-                            .field("Stacktrace.id")
-                            // 'execution_hint: map' skips the slow building of ordinals that we don't need.
-                            // Especially with high cardinality fields, this makes aggregations really slow.
-                            .executionHint("map")
-                            .subAggregation(new SumAggregationBuilder("count").field("Stacktrace.count"))
-                    )
+                    .subAggregation(groupByStackTraceId)
             )
             .addAggregation(new SumAggregationBuilder("total_count").field("Stacktrace.count"))
             .execute(handleEventsGroupedByStackTrace(submitTask, client, responseBuilder, submitListener, searchResponse -> {
@@ -370,6 +408,14 @@ public class TransportGetStackTracesAction extends TransportAction<GetStackTrace
                             stackTraceEvents.put(stackTraceID, event);
                         }
                         event.count += finalCount;
+                        if (request.getAggregationField() != null) {
+                            Terms subGroup = stacktraceBucket.getAggregations().get(CUSTOM_EVENT_SUB_AGGREGATION_NAME);
+                            for (Terms.Bucket b : subGroup.getBuckets()) {
+                                String subGroupName = b.getKeyAsString();
+                                long subGroupCount = event.subGroups.getOrDefault(subGroupName, 0L);
+                                event.subGroups.put(subGroupName, subGroupCount + b.getDocCount());
+                            }
+                        }
                     }
                 }
                 responseBuilder.setTotalSamples(totalFinalCount);

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetTopNFunctionsAction.java

@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiling;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.client.internal.ParentTaskAssigningClient;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class TransportGetTopNFunctionsAction extends TransportAction<GetStackTracesRequest, GetTopNFunctionsResponse> {
+    private static final Logger log = LogManager.getLogger(TransportGetTopNFunctionsAction.class);
+    private final NodeClient nodeClient;
+    private final TransportService transportService;
+
+    @Inject
+    public TransportGetTopNFunctionsAction(NodeClient nodeClient, TransportService transportService, ActionFilters actionFilters) {
+        super(GetTopNFunctionsAction.NAME, actionFilters, transportService.getTaskManager());
+        this.nodeClient = nodeClient;
+        this.transportService = transportService;
+    }
+
+    @Override
+    protected void doExecute(Task task, GetStackTracesRequest request, ActionListener<GetTopNFunctionsResponse> listener) {
+        Client client = new ParentTaskAssigningClient(this.nodeClient, transportService.getLocalNode(), task);
+        StopWatch watch = new StopWatch("getTopNFunctionsAction");
+        client.execute(GetStackTracesAction.INSTANCE, request, ActionListener.wrap(searchResponse -> {
+            StopWatch processingWatch = new StopWatch("Processing response");
+            GetTopNFunctionsResponse topNFunctionsResponse = buildTopNFunctions(searchResponse, request.getLimit());
+            log.debug(() -> watch.report() + " " + processingWatch.report());
+            listener.onResponse(topNFunctionsResponse);
+        }, listener::onFailure));
+    }
+
+    static GetTopNFunctionsResponse buildTopNFunctions(GetStackTracesResponse response, Integer limit) {
+        TopNFunctionsBuilder builder = new TopNFunctionsBuilder(limit);
+        if (response.getTotalFrames() == 0) {
+            return builder.build();
+        }
+
+        for (StackTrace stackTrace : response.getStackTraces().values()) {
+            Set<String> frameGroupsPerStackTrace = new HashSet<>();
+            long samples = stackTrace.count;
+            double annualCO2Tons = stackTrace.annualCO2Tons;
+            double annualCostsUSD = stackTrace.annualCostsUSD;
+
+            int frameCount = stackTrace.frameIds.length;
+            for (int i = 0; i < frameCount; i++) {
+                String frameId = stackTrace.frameIds[i];
+                String fileId = stackTrace.fileIds[i];
+                int frameType = stackTrace.typeIds[i];
+                int addressOrLine = stackTrace.addressOrLines[i];
+                StackFrame stackFrame = response.getStackFrames().getOrDefault(frameId, StackFrame.EMPTY_STACKFRAME);
+                String executable = response.getExecutables().getOrDefault(fileId, "");
+
+                final boolean isLeafFrame = i == frameCount - 1;
+                stackFrame.forEach(frame -> {
+                    // The samples associated with a frame provide the total number of
+                    // traces in which that frame has appeared at least once. However, a
+                    // frame may appear multiple times in a trace, and thus to avoid
+                    // counting it multiple times we need to record the frames seen so
+                    // far in each trace. Instead of using the entire frame information
+                    // to determine if a frame has already been seen within a given
+                    // stacktrace, we use the frame group ID for a frame.
+                    String frameGroupId = FrameGroupID.create(fileId, addressOrLine, executable, frame.fileName(), frame.functionName());
+                    if (builder.isExists(frameGroupId) == false) {
+                        builder.addTopNFunction(
+                            new TopNFunction(
+                                frameGroupId,
+                                frameType,
+                                frame.inline(),
+                                addressOrLine,
+                                frame.functionName(),
+                                frame.fileName(),
+                                frame.lineNumber(),
+                                executable
+                            )
+                        );
+                    }
+                    TopNFunction current = builder.getTopNFunction(frameGroupId);
+                    if (stackTrace.subGroups != null) {
+                        current.addSubGroups(stackTrace.subGroups);
+                    }
+                    if (frameGroupsPerStackTrace.contains(frameGroupId) == false) {
+                        frameGroupsPerStackTrace.add(frameGroupId);
+                        current.addTotalCount(samples);
+                        current.addTotalAnnualCO2Tons(annualCO2Tons);
+                        current.addTotalAnnualCostsUSD(annualCostsUSD);
+
+                    }
+                    if (isLeafFrame && frame.last()) {
+                        // Leaf frame: sum up counts for self CPU.
+                        current.addSelfCount(samples);
+                        current.addSelfAnnualCO2Tons(annualCO2Tons);
+                        current.addSelfAnnualCostsUSD(annualCostsUSD);
+
+                    }
+                });
+            }
+        }
+
+        return builder.build();
+    }
+
+    private static class TopNFunctionsBuilder {
+        private final Integer limit;
+        private final HashMap<String, TopNFunction> topNFunctions;
+
+        TopNFunctionsBuilder(Integer limit) {
+            this.limit = limit;
+            this.topNFunctions = new HashMap<>();
+        }
+
+        public GetTopNFunctionsResponse build() {
+            List<TopNFunction> functions = new ArrayList<>(topNFunctions.values());
+            functions.sort(Collections.reverseOrder());
+            long sumSelfCount = 0;
+            long sumTotalCount = 0;
+            for (int i = 0; i < functions.size(); i++) {
+                TopNFunction topNFunction = functions.get(i);
+                topNFunction.setRank(i + 1);
+                sumSelfCount += topNFunction.getSelfCount();
+                sumTotalCount += topNFunction.getTotalCount();
+            }
+            // limit at the end so global stats are independent of the limit
+            if (limit != null && limit > 0) {
+                functions = functions.subList(0, limit);
+            }
+            return new GetTopNFunctionsResponse(sumSelfCount, sumTotalCount, functions);
+        }
+
+        public boolean isExists(String frameGroupID) {
+            return this.topNFunctions.containsKey(frameGroupID);
+        }
+
+        public TopNFunction getTopNFunction(String frameGroupID) {
+            return this.topNFunctions.get(frameGroupID);
+        }
+
+        public void addTopNFunction(TopNFunction topNFunction) {
+            this.topNFunctions.put(topNFunction.getId(), topNFunction);
+        }
+    }
+}

x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/GetStackTracesRequestTests.java

@@ -254,6 +254,7 @@ public class GetStackTracesRequestTests extends ESTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         List<String> validationErrors = request.validate().validationErrors();
@@ -274,6 +275,7 @@ public class GetStackTracesRequestTests extends ESTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         assertNull("Expecting no validation errors", request.validate());
@@ -292,6 +294,7 @@ public class GetStackTracesRequestTests extends ESTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         List<String> validationErrors = request.validate().validationErrors();
@@ -312,6 +315,7 @@ public class GetStackTracesRequestTests extends ESTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         List<String> validationErrors = request.validate().validationErrors();
@@ -333,6 +337,7 @@ public class GetStackTracesRequestTests extends ESTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         String[] indices = request.indices();
@@ -341,7 +346,21 @@ public class GetStackTracesRequestTests extends ESTestCase {
     }
 
     public void testConsidersDefaultIndicesInRelatedIndices() {
-        GetStackTracesRequest request = new GetStackTracesRequest(1, 1.0d, 1.0d, 1.0d, null, null, null, null, null, null, null, null);
+        GetStackTracesRequest request = new GetStackTracesRequest(
+            1,
+            1.0d,
+            1.0d,
+            1.0d,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null
+        );
         String[] indices = request.indices();
         assertEquals(15, indices.length);
     }

x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/ResamplerTests.java

@@ -42,6 +42,7 @@ public class ResamplerTests extends ESTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         request.setAdjustSampleCount(false);
@@ -70,6 +71,7 @@ public class ResamplerTests extends ESTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         request.setAdjustSampleCount(true);
@@ -98,6 +100,7 @@ public class ResamplerTests extends ESTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         request.setAdjustSampleCount(false);
@@ -129,6 +132,7 @@ public class ResamplerTests extends ESTestCase {
             null,
             null,
             null,
+            null,
             null
         );
 
@@ -157,6 +161,7 @@ public class ResamplerTests extends ESTestCase {
             null,
             null,
             null,
+            null,
             null
         );
         request.setAdjustSampleCount(true);

x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/StackFrameTests.java

@@ -62,6 +62,5 @@ public class StackFrameTests extends ESTestCase {
             frame,
             (o -> new StackFrame(o.fileName, o.functionName, o.functionOffset, o.lineNumber))
         );
-
     }
 }

x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/TopNFunctionTests.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiling;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class TopNFunctionTests extends ESTestCase {
+    public void testToXContent() throws IOException {
+        String fileID = "6tVKI4mSYDEJ-ABAIpYXcg";
+        int frameType = 1;
+        boolean inline = false;
+        int addressOrLine = 23;
+        String functionName = "PyDict_GetItemWithError";
+        String sourceFilename = "/build/python3.9-RNBry6/python3.9-3.9.2/Objects/dictobject.c";
+        int sourceLine = 1456;
+        String exeFilename = "python3.9";
+
+        String frameGroupID = FrameGroupID.create(fileID, addressOrLine, exeFilename, sourceFilename, functionName);
+
+        XContentType contentType = randomFrom(XContentType.values());
+        XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType)
+            .startObject()
+            .field("id", frameGroupID)
+            .field("rank", 1)
+            .startObject("frame")
+            .field("frame_type", frameType)
+            .field("inline", inline)
+            .field("address_or_line", addressOrLine)
+            .field("function_name", functionName)
+            .field("file_name", sourceFilename)
+            .field("line_number", sourceLine)
+            .field("executable_file_name", exeFilename)
+            .endObject()
+            .field("sub_groups", Map.of("basket", 7L))
+            .field("self_count", 1)
+            .field("total_count", 10)
+            .field("self_annual_co2_tons")
+            .rawValue("2.2000")
+            .field("total_annual_co2_tons")
+            .rawValue("22.0000")
+            .field("self_annual_costs_usd", "12.0000")
+            .field("total_annual_costs_usd", "120.0000")
+            .endObject();
+
+        XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType);
+        TopNFunction topNFunction = new TopNFunction(
+            frameGroupID,
+            1,
+            frameType,
+            inline,
+            addressOrLine,
+            functionName,
+            sourceFilename,
+            sourceLine,
+            exeFilename,
+            1,
+            10,
+            2.2d,
+            22.0d,
+            12.0d,
+            120.0d,
+            Map.of("basket", 7L)
+        );
+        topNFunction.toXContent(actualRequest, ToXContent.EMPTY_PARAMS);
+
+        assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType);
+    }
+
+    public void testEquality() {
+        String fileID = "6tVKI4mSYDEJ-ABAIpYXcg";
+        int frameType = 1;
+        boolean inline = false;
+        int addressOrLine = 23;
+        String functionName = "PyDict_GetItemWithError";
+        String sourceFilename = "/build/python3.9-RNBry6/python3.9-3.9.2/Objects/dictobject.c";
+        int sourceLine = 1456;
+        String exeFilename = "python3.9";
+
+        String frameGroupID = FrameGroupID.create(fileID, addressOrLine, exeFilename, sourceFilename, functionName);
+
+        TopNFunction topNFunction = new TopNFunction(
+            frameGroupID,
+            1,
+            frameType,
+            inline,
+            addressOrLine,
+            functionName,
+            sourceFilename,
+            sourceLine,
+            exeFilename,
+            1,
+            10,
+            2.0d,
+            4.0d,
+            23.2d,
+            12.0d,
+            Map.of("checkout", 4L, "basket", 12L)
+        );
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(topNFunction, (TopNFunction::clone));
+    }
+}

x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/TransportGetFlamegraphActionTests.java

@@ -116,7 +116,6 @@ public class TransportGetFlamegraphActionTests extends ESTestCase {
         assertEquals(1L, response.getSelfCPU());
         assertEquals(10L, response.getTotalCPU());
         assertEquals(1L, response.getTotalSamples());
-
     }
 
     public void testCreateEmptyFlamegraphWithRootNode() {

x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/TransportGetTopNFunctionsActionTests.java

@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiling;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class TransportGetTopNFunctionsActionTests extends ESTestCase {
+    public void testCreateAllTopNFunctions() {
+        GetStackTracesResponse stacktraces = new GetStackTracesResponse(
+            Map.of(
+                "2buqP1GpF-TXYmL4USW8gA",
+                new StackTrace(
+                    new int[] { 12784352, 19334053, 19336161, 18795859, 18622708, 18619213, 12989721, 13658842, 16339645 },
+                    new String[] {
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w" },
+                    new String[] {
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAAAwxLg",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAABJwOl",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAABJwvh",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAABHs1T",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAABHCj0",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAABHBtN",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAAAxjUZ",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAAA0Gra",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAAA-VK9" },
+                    new int[] { 3, 3, 3, 3, 3, 3, 3, 3, 3 },
+                    0.3d,
+                    2.7d,
+                    1
+                )
+            ),
+            Map.of(),
+            Map.of("fr28zxcZ2UDasxYuu6dV-w", "containerd"),
+            Map.of("2buqP1GpF-TXYmL4USW8gA", new TraceEvent("2buqP1GpF-TXYmL4USW8gA", 1L)),
+            9,
+            1.0d,
+            1
+        );
+
+        GetTopNFunctionsResponse response = TransportGetTopNFunctionsAction.buildTopNFunctions(stacktraces, null);
+        assertNotNull(response);
+        assertEquals(1, response.getSelfCount());
+        assertEquals(9, response.getTotalCount());
+
+        List<TopNFunction> topNFunctions = response.getTopN();
+        assertNotNull(topNFunctions);
+        assertEquals(9, topNFunctions.size());
+
+        assertEquals(
+            List.of(
+                topN("178196121", 1, 16339645, 1L, 1L, 0.3d, 0.3d, 2.7d, 2.7d),
+                topN("181192637", 2, 19336161, 0L, 1L, 0.0d, 0.3d, 0.0d, 2.7d),
+                topN("181190529", 3, 19334053, 0L, 1L, 0.0d, 0.3d, 0.0d, 2.7d),
+                topN("180652335", 4, 18795859, 0L, 1L, 0.0d, 0.3d, 0.0d, 2.7d),
+                topN("180479184", 5, 18622708, 0L, 1L, 0.0d, 0.3d, 0.0d, 2.7d),
+                topN("180475689", 6, 18619213, 0L, 1L, 0.0d, 0.3d, 0.0d, 2.7d),
+                topN("175515318", 7, 13658842, 0L, 1L, 0.0d, 0.3d, 0.0d, 2.7d),
+                topN("174846197", 8, 12989721, 0L, 1L, 0.0d, 0.3d, 0.0d, 2.7d),
+                topN("174640828", 9, 12784352, 0L, 1L, 0.0d, 0.3d, 0.0d, 2.7d)
+            ),
+            topNFunctions
+        );
+    }
+
+    public void testCreateTopNFunctionsWithLimit() {
+        GetStackTracesResponse stacktraces = new GetStackTracesResponse(
+            Map.of(
+                "2buqP1GpF-TXYmL4USW8gA",
+                new StackTrace(
+                    new int[] { 12784352, 19334053, 19336161, 18795859, 18622708, 18619213, 12989721, 13658842, 16339645 },
+                    new String[] {
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w",
+                        "fr28zxcZ2UDasxYuu6dV-w" },
+                    new String[] {
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAAAwxLg",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAABJwOl",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAABJwvh",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAABHs1T",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAABHCj0",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAABHBtN",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAAAxjUZ",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAAA0Gra",
+                        "fr28zxcZ2UDasxYuu6dV-wAAAAAA-VK9" },
+                    new int[] { 3, 3, 3, 3, 3, 3, 3, 3, 3 },
+                    0.3d,
+                    2.7d,
+                    1
+                )
+            ),
+            Map.of(),
+            Map.of("fr28zxcZ2UDasxYuu6dV-w", "containerd"),
+            Map.of("2buqP1GpF-TXYmL4USW8gA", new TraceEvent("2buqP1GpF-TXYmL4USW8gA", 1L)),
+            9,
+            1.0d,
+            1
+        );
+
+        GetTopNFunctionsResponse response = TransportGetTopNFunctionsAction.buildTopNFunctions(stacktraces, 3);
+        assertNotNull(response);
+        assertEquals(1, response.getSelfCount());
+        assertEquals(9, response.getTotalCount());
+
+        List<TopNFunction> topNFunctions = response.getTopN();
+        assertNotNull(topNFunctions);
+        assertEquals(3, topNFunctions.size());
+
+        assertEquals(
+            List.of(
+                topN("178196121", 1, 16339645, 1L, 1L, 0.3d, 0.3d, 2.7d, 2.7d),
+                topN("181192637", 2, 19336161, 0L, 1L, 0.0d, 0.3d, 0.0d, 2.7d),
+                topN("181190529", 3, 19334053, 0L, 1L, 0.0d, 0.3d, 0.0d, 2.7d)
+            ),
+            topNFunctions
+        );
+    }
+
+    private TopNFunction topN(
+        String id,
+        int rank,
+        int addressOrLine,
+        long exclusiveCount,
+        long inclusiveCount,
+        double annualCO2TonsExclusive,
+        double annualCO2TonsInclusive,
+        double annualCostsUSDExclusive,
+        double annualCostsUSDInclusive
+    ) {
+        return new TopNFunction(
+            id,
+            rank,
+            3,
+            false,
+            addressOrLine,
+            "",
+            "",
+            0,
+            "containerd",
+            exclusiveCount,
+            inclusiveCount,
+            annualCO2TonsExclusive,
+            annualCO2TonsInclusive,
+            annualCostsUSDExclusive,
+            annualCostsUSDInclusive,
+            Collections.emptyMap()
+        );
+    }
+
+    public void testCreateEmptyTopNFunctions() {
+        GetStackTracesResponse stacktraces = new GetStackTracesResponse(Map.of(), Map.of(), Map.of(), Map.of(), 0, 1.0d, 0);
+        GetTopNFunctionsResponse response = TransportGetTopNFunctionsAction.buildTopNFunctions(stacktraces, null);
+        assertNotNull(response);
+        assertEquals(0, response.getSelfCount());
+        assertEquals(0, response.getTotalCount());
+
+        List<TopNFunction> topNFunctions = response.getTopN();
+        assertNotNull(topNFunctions);
+        assertEquals(0, topNFunctions.size());
+    }
+}

x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java

@@ -549,6 +549,7 @@ public class Constants {
         "indices:data/read/open_point_in_time",
         "indices:data/read/profiling/stack_traces",
         "indices:data/read/profiling/flamegraph",
+        "indices:data/read/profiling/topn/functions",
         "indices:data/read/rank_eval",
         "indices:data/read/scroll",
         "indices:data/read/scroll/clear",

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml

@@ -217,3 +217,67 @@ teardown:
             }
           }
   - match: { Size: 47}
+
+---
+"Test topN functions from profiling-events":
+  - skip:
+      version: "- 8.13.99"
+      reason: "the topN functions API was added in 8.14.0"
+
+  - do:
+      profiling.topn_functions:
+        body: >
+          {
+            "sample_size": 20000,
+            "requested_duration": 86400,
+            "limit": 10,
+            "query": {
+              "bool": {
+                "filter": [
+                  {
+                    "range": {
+                      "@timestamp": {
+                        "gte": "2023-11-20",
+                        "lt": "2023-11-21",
+                        "format": "yyyy-MM-dd"
+                      }
+                    }
+                  }
+                ]
+              }
+            }
+          }
+  - length: { topn: 10}
+
+---
+"Test topN functions from test-events":
+  - skip:
+      version: "- 8.13.99"
+      reason: "the topN functions API was added in 8.14.0"
+
+  - do:
+      profiling.topn_functions:
+        body: >
+          {
+            "sample_size": 20000,
+            "indices": ["test-event*"],
+            "stacktrace_ids_field": "events",
+            "requested_duration": 86400,
+            "limit": 10,
+            "query": {
+              "bool": {
+                "filter": [
+                  {
+                    "range": {
+                      "@timestamp": {
+                        "gte": "2023-11-20",
+                        "lt": "2023-11-21",
+                        "format": "yyyy-MM-dd"
+                      }
+                    }
+                  }
+                ]
+              }
+            }
+          }
+  - length: { topn: 10}