[Profiling] Speed up serialization of flamegraph (#105779)

The response of the flamegraph is quite large: A typical response can
easily reach 50MB (uncompressed). In order to reduce memory pressure and
also to start sending the response sooner, we chunk the response.
However, this leads to many chunks that are very small and lead to high
overhead. In our experiments, just the serialization takes more than
500ms.

With this commit we take the following measures:

1. We split the response into chunks only when it makes sense and
   otherwise send one larger chunk.
2. Serialization of doubles is very expensive: Just the serialization of
   annual CO2 tons takes around 80ms in our test setup. Therefore, we
apply a custom serialization that is both faster than the builtin
serialization as well reduces the amount of bytes sent over the wire
because we round to four decimal places (which is more than sufficient for 
our purposes).

docs/changelog/105779.yaml

@@ -0,0 +1,5 @@
+pr: 105779
+summary: "[Profiling] Speed up serialization of flamegraph"
+area: Application
+type: enhancement
+issues: []

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentGenerator.java

@@ -496,6 +496,20 @@ public class JsonXContentGenerator implements XContentGenerator {
         }
     }
 
+    @Override
+    public void writeRawValue(String value) throws IOException {
+        try {
+            if (supportsRawWrites()) {
+                generator.writeRaw(value);
+            } else {
+                // fallback to a regular string for formats that don't allow writing the value as is
+                generator.writeString(value);
+            }
+        } catch (JsonGenerationException e) {
+            throw new XContentGenerationException(e);
+        }
+    }
+
     private boolean mayWriteRawData(XContentType contentType) {
         // When the current generator is filtered (ie filter != null)
         // or the content is in a different format than the current generator,

libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java

@@ -1212,6 +1212,14 @@ public final class XContentBuilder implements Closeable, Flushable {
         return this;
     }
 
+    /**
+     * Writes a value with the source coming directly from a pre-rendered string representation
+     */
+    public XContentBuilder rawValue(String value) throws IOException {
+        generator.writeRawValue(value);
+        return this;
+    }
+
     public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOException {
         generator.copyCurrentStructure(parser);
         return this;

libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerator.java

@@ -105,6 +105,11 @@ public interface XContentGenerator extends Closeable, Flushable {
      */
     void writeRawValue(InputStream value, XContentType xContentType) throws IOException;
 
+    /**
+     * Writes a raw value taken from a pre-rendered string representation
+     */
+    void writeRawValue(String value) throws IOException;
+
     void copyCurrentStructure(XContentParser parser) throws IOException;
 
     /**

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetFlamegraphResponse.java

@@ -191,27 +191,64 @@ public class GetFlamegraphResponse extends ActionResponse implements ChunkedToXC
             ChunkedToXContentHelper.array("ExeFilename", Iterators.map(fileNames.iterator(), e -> (b, p) -> b.value(e))),
             ChunkedToXContentHelper.array("AddressOrLine", Iterators.map(addressOrLines.iterator(), e -> (b, p) -> b.value(e))),
             ChunkedToXContentHelper.array("FunctionName", Iterators.map(functionNames.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.array("FunctionOffset", Iterators.map(functionOffsets.iterator(), e -> (b, p) -> b.value(e))),
+            ChunkedToXContentHelper.singleChunk((b, p) -> {
+                b.startArray("FunctionOffset");
+                for (int functionOffset : functionOffsets) {
+                    b.value(functionOffset);
+                }
+                return b.endArray();
+            }),
             ChunkedToXContentHelper.array("SourceFilename", Iterators.map(sourceFileNames.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.array("SourceLine", Iterators.map(sourceLines.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.array("CountInclusive", Iterators.map(countInclusive.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.array("CountExclusive", Iterators.map(countExclusive.iterator(), e -> (b, p) -> b.value(e))),
-            ChunkedToXContentHelper.array(
-                "AnnualCO2TonsInclusive",
-                Iterators.map(annualCO2TonsInclusive.iterator(), e -> (b, p) -> b.value(e))
-            ),
-            ChunkedToXContentHelper.array(
-                "AnnualCO2TonsExclusive",
-                Iterators.map(annualCO2TonsExclusive.iterator(), e -> (b, p) -> b.value(e))
-            ),
-            ChunkedToXContentHelper.array(
-                "AnnualCostsUSDInclusive",
-                Iterators.map(annualCostsUSDInclusive.iterator(), e -> (b, p) -> b.value(e))
-            ),
-            ChunkedToXContentHelper.array(
-                "AnnualCostsUSDExclusive",
-                Iterators.map(annualCostsUSDExclusive.iterator(), e -> (b, p) -> b.value(e))
-            ),
+            ChunkedToXContentHelper.singleChunk((b, p) -> {
+                b.startArray("SourceLine");
+                for (int sourceLine : sourceLines) {
+                    b.value(sourceLine);
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.singleChunk((b, p) -> {
+                b.startArray("CountInclusive");
+                for (long countInclusive : countInclusive) {
+                    b.value(countInclusive);
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.singleChunk((b, p) -> {
+                b.startArray("CountExclusive");
+                for (long c : countExclusive) {
+                    b.value(c);
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.singleChunk((b, p) -> {
+                b.startArray("AnnualCO2TonsInclusive");
+                for (double co2Tons : annualCO2TonsInclusive) {
+                    // write as raw value - we need direct control over the output representation (here: limit to 4 decimal places)
+                    b.rawValue(NumberUtils.doubleToString(co2Tons));
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.singleChunk((b, p) -> {
+                b.startArray("AnnualCO2TonsExclusive");
+                for (double co2Tons : annualCO2TonsExclusive) {
+                    b.rawValue(NumberUtils.doubleToString(co2Tons));
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.singleChunk((b, p) -> {
+                b.startArray("AnnualCostsUSDInclusive");
+                for (double costs : annualCostsUSDInclusive) {
+                    b.rawValue(NumberUtils.doubleToString(costs));
+                }
+                return b.endArray();
+            }),
+            ChunkedToXContentHelper.singleChunk((b, p) -> {
+                b.startArray("AnnualCostsUSDExclusive");
+                for (double costs : annualCostsUSDExclusive) {
+                    b.rawValue(NumberUtils.doubleToString(costs));
+                }
+                return b.endArray();
+            }),
             Iterators.single((b, p) -> b.field("Size", size)),
             Iterators.single((b, p) -> b.field("SamplingRate", samplingRate)),
             Iterators.single((b, p) -> b.field("SelfCPU", selfCPU)),

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/NumberUtils.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiling;
+
+final class NumberUtils {
+    private NumberUtils() {
+        // no instances intended
+    }
+
+    /**
+     * Converts a positive double number to a string.
+     *
+     * @param value The double value.
+     * @return The corresponding string representation rounded to four fractional digits.
+     */
+    public static String doubleToString(double value) {
+        if (value < 0.0001d) {
+            return "0";
+        }
+        StringBuilder sb = new StringBuilder();
+        int i = (int) value;
+        int f = (int) ((value - i) * 10000.0d + 0.5d);
+        sb.append(i);
+        sb.append(".");
+        if (f < 10) {
+            sb.append("000");
+        } else if (f < 100) {
+            sb.append("00");
+        } else if (f < 1000) {
+            sb.append("0");
+        }
+        sb.append(f);
+        return sb.toString();
+    }
+}

x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/NumberUtilsTests.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiling;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class NumberUtilsTests extends ESTestCase {
+    public void testConvertNumberToString() {
+        assertEquals("872.6182", NumberUtils.doubleToString(872.6181989583333d));
+        assertEquals("1222.1833", NumberUtils.doubleToString(1222.18325d));
+        assertEquals("1222.1832", NumberUtils.doubleToString(1222.18324d));
+        assertEquals("1.0013", NumberUtils.doubleToString(1.0013d));
+        assertEquals("10.0220", NumberUtils.doubleToString(10.022d));
+        assertEquals("222.0000", NumberUtils.doubleToString(222.0d));
+        assertEquals("0.0001", NumberUtils.doubleToString(0.0001d));
+    }
+
+    public void testConvertZeroToString() {
+        assertEquals("0", NumberUtils.doubleToString(0.0d));
+        assertEquals("0", NumberUtils.doubleToString(0.00009d));
+    }
+}