Browse Source

Add min support to exponential histograms (#133639)

Jonas Kunz 1 month ago
parent
commit
9423dcbcf1
16 changed files with 373 additions and 44 deletions
  1. 1 1
      benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java
  2. 5 0
      libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/EmptyExponentialHistogram.java
  3. 7 0
      libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogram.java
  4. 12 3
      libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramGenerator.java
  5. 11 1
      libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java
  6. 43 0
      libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtils.java
  7. 4 0
      libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContent.java
  8. 11 0
      libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/FixedCapacityExponentialHistogram.java
  9. 8 2
      libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMergerTests.java
  10. 84 1
      libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtilsTests.java
  11. 8 3
      libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContentTests.java
  12. 11 2
      x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/CompressedExponentialHistogram.java
  13. 46 3
      x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java
  14. 55 19
      x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/IndexWithCount.java
  15. 50 6
      x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java
  16. 17 3
      x-pack/plugin/mapper-exponential-histogram/src/yamlRestTest/resources/rest-api-spec/test/10_synthetic_source.yml

+ 1 - 1
benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -130,7 +130,7 @@ public class ExponentialHistogramMergeBench {
             CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
             CompressedExponentialHistogram result = new CompressedExponentialHistogram();
             BytesRef data = histoBytes.bytes().toBytesRef();
-            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), data);
+            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), data);
             return result;
         } catch (IOException e) {
             throw new RuntimeException(e);

+ 5 - 0
libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/EmptyExponentialHistogram.java

@@ -82,6 +82,11 @@ class EmptyExponentialHistogram implements ReleasableExponentialHistogram {
         return 0;
     }
 
+    @Override
+    public double min() {
+        return Double.NaN;
+    }
+
     @Override
     public long ramBytesUsed() {
         return 0;

+ 7 - 0
libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogram.java

@@ -102,6 +102,13 @@ public interface ExponentialHistogram extends Accountable {
      */
     double sum();
 
+    /**
+     * Returns minimum of all values represented by this histogram.
+     *
+     * @return the minimum, NaN for empty histograms
+     */
+    double min();
+
     /**
      * Represents a bucket range of an {@link ExponentialHistogram}, either the positive or the negative range.
      */

+ 12 - 3
libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramGenerator.java

@@ -123,7 +123,9 @@ public class ExponentialHistogramGenerator implements Accountable, Releasable {
         }
 
         valueBuffer.reset();
-        valueBuffer.setSum(rawValuesSum());
+        Aggregates aggregates = rawValuesAggregates();
+        valueBuffer.setSum(aggregates.sum());
+        valueBuffer.setMin(aggregates.min());
         int scale = valueBuffer.scale();
 
         // Buckets must be provided with their indices in ascending order.
@@ -162,12 +164,17 @@ public class ExponentialHistogramGenerator implements Accountable, Releasable {
         valueCount = 0;
     }
 
-    private double rawValuesSum() {
+    private Aggregates rawValuesAggregates() {
+        if (valueCount == 0) {
+            return new Aggregates(0, Double.NaN);
+        }
         double sum = 0;
+        double min = Double.MAX_VALUE;
         for (int i = 0; i < valueCount; i++) {
             sum += rawValueBuffer[i];
+            min = Math.min(min, rawValueBuffer[i]);
         }
-        return sum;
+        return new Aggregates(sum, min);
     }
 
     private static long estimateBaseSize(int numBuckets) {
@@ -190,4 +197,6 @@ public class ExponentialHistogramGenerator implements Accountable, Releasable {
             circuitBreaker.adjustBreaker(-estimateBaseSize(rawValueBuffer.length));
         }
     }
+
+    private record Aggregates(double sum, double min) {}
 }

+ 11 - 1
libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java

@@ -151,7 +151,7 @@ public class ExponentialHistogramMerger implements Accountable, Releasable {
         }
         buffer.setZeroBucket(zeroBucket);
         buffer.setSum(a.sum() + b.sum());
-
+        buffer.setMin(nanAwareMin(a.min(), b.min()));
         // We attempt to bring everything to the scale of A.
         // This might involve increasing the scale for B, which would increase its indices.
         // We need to ensure that we do not exceed MAX_INDEX / MIN_INDEX in this case.
@@ -231,4 +231,14 @@ public class ExponentialHistogramMerger implements Accountable, Releasable {
         return overflowCount;
     }
 
+    private static double nanAwareMin(double a, double b) {
+        if (Double.isNaN(a)) {
+            return b;
+        }
+        if (Double.isNaN(b)) {
+            return a;
+        }
+        return Math.min(a, b);
+    }
+
 }

+ 43 - 0
libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtils.java

@@ -21,6 +21,9 @@
 
 package org.elasticsearch.exponentialhistogram;
 
+import java.util.OptionalDouble;
+import java.util.OptionalLong;
+
 public class ExponentialHistogramUtils {
 
     /**
@@ -59,4 +62,44 @@ public class ExponentialHistogramUtils {
         }
         return sum;
     }
+
+    /**
+     * Estimates the minimum value of the histogram based on the populated buckets.
+     * The returned value is guaranteed to be less than or equal to the exact minimum value of the histogram values.
+     * If the histogram is empty, an empty Optional is returned.
+     *
+     * Note that this method can return +-Infinity if the histogram bucket boundaries are not representable in a double.
+     *
+     * @param zeroBucket the zero bucket of the histogram
+     * @param negativeBuckets the negative buckets of the histogram
+     * @param positiveBuckets the positive buckets of the histogram
+     * @return the estimated minimum
+     */
+    public static OptionalDouble estimateMin(
+        ZeroBucket zeroBucket,
+        ExponentialHistogram.Buckets negativeBuckets,
+        ExponentialHistogram.Buckets positiveBuckets
+    ) {
+        int scale = negativeBuckets.iterator().scale();
+        assert scale == positiveBuckets.iterator().scale();
+
+        OptionalLong negativeMaxIndex = negativeBuckets.maxBucketIndex();
+        if (negativeMaxIndex.isPresent()) {
+            return OptionalDouble.of(-ExponentialScaleUtils.getUpperBucketBoundary(negativeMaxIndex.getAsLong(), scale));
+        }
+
+        if (zeroBucket.count() > 0) {
+            if (zeroBucket.zeroThreshold() == 0.0) {
+                // avoid negative zero
+                return OptionalDouble.of(0.0);
+            }
+            return OptionalDouble.of(-zeroBucket.zeroThreshold());
+        }
+
+        BucketIterator positiveBucketsIt = positiveBuckets.iterator();
+        if (positiveBucketsIt.hasNext()) {
+            return OptionalDouble.of(ExponentialScaleUtils.getLowerBucketBoundary(positiveBucketsIt.peekIndex(), scale));
+        }
+        return OptionalDouble.empty();
+    }
 }

+ 4 - 0
libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContent.java

@@ -32,6 +32,7 @@ public class ExponentialHistogramXContent {
 
     public static final String SCALE_FIELD = "scale";
     public static final String SUM_FIELD = "sum";
+    public static final String MIN_FIELD = "min";
     public static final String ZERO_FIELD = "zero";
     public static final String ZERO_COUNT_FIELD = "count";
     public static final String ZERO_THRESHOLD_FIELD = "threshold";
@@ -51,6 +52,9 @@ public class ExponentialHistogramXContent {
 
         builder.field(SCALE_FIELD, histogram.scale());
         builder.field(SUM_FIELD, histogram.sum());
+        if (Double.isNaN(histogram.min()) == false) {
+            builder.field(MIN_FIELD, histogram.min());
+        }
         double zeroThreshold = histogram.zeroBucket().zeroThreshold();
         long zeroCount = histogram.zeroBucket().count();
 

+ 11 - 0
libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/FixedCapacityExponentialHistogram.java

@@ -54,6 +54,7 @@ final class FixedCapacityExponentialHistogram implements ReleasableExponentialHi
     private final Buckets positiveBuckets = new Buckets(true);
 
     private double sum;
+    private double min;
 
     private final ExponentialHistogramCircuitBreaker circuitBreaker;
     private boolean closed = false;
@@ -81,6 +82,7 @@ final class FixedCapacityExponentialHistogram implements ReleasableExponentialHi
      */
     void reset() {
         sum = 0;
+        min = Double.NaN;
         setZeroBucket(ZeroBucket.minimalEmpty());
         resetBuckets(MAX_SCALE);
     }
@@ -122,6 +124,15 @@ final class FixedCapacityExponentialHistogram implements ReleasableExponentialHi
         this.sum = sum;
     }
 
+    @Override
+    public double min() {
+        return min;
+    }
+
+    void setMin(double min) {
+        this.min = min;
+    }
+
     /**
      * Attempts to add a bucket to the positive or negative range of this histogram.
      * <br>

+ 8 - 2
libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMergerTests.java

@@ -29,6 +29,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
+import java.util.stream.DoubleStream;
 import java.util.stream.IntStream;
 
 import static org.elasticsearch.exponentialhistogram.ExponentialHistogram.MAX_INDEX;
@@ -106,19 +107,24 @@ public class ExponentialHistogramMergerTests extends ExponentialHistogramTestCas
         assertThat(posBuckets.hasNext(), equalTo(false));
     }
 
-    public void testSumCorrectness() {
+    public void testAggregatesCorrectness() {
         double[] firstValues = randomDoubles(100).map(val -> val * 2 - 1).toArray();
         double[] secondValues = randomDoubles(50).map(val -> val * 2 - 1).toArray();
         double correctSum = Arrays.stream(firstValues).sum() + Arrays.stream(secondValues).sum();
+        double correctMin = DoubleStream.concat(Arrays.stream(firstValues), Arrays.stream(secondValues)).min().getAsDouble();
         try (
+            // Merge some empty histograms too to test that code path
             ReleasableExponentialHistogram merged = ExponentialHistogram.merge(
                 2,
                 breaker(),
+                ExponentialHistogram.empty(),
                 createAutoReleasedHistogram(10, firstValues),
-                createAutoReleasedHistogram(20, secondValues)
+                createAutoReleasedHistogram(20, secondValues),
+                ExponentialHistogram.empty()
             )
         ) {
             assertThat(merged.sum(), closeTo(correctSum, 0.000001));
+            assertThat(merged.min(), equalTo(correctMin));
         }
     }
 

+ 84 - 1
libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtilsTests.java

@@ -21,6 +21,8 @@
 
 package org.elasticsearch.exponentialhistogram;
 
+import java.util.OptionalDouble;
+
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -57,7 +59,7 @@ public class ExponentialHistogramUtilsTests extends ExponentialHistogramTestCase
         }
     }
 
-    public void testInfinityHandling() {
+    public void testSumInfinityHandling() {
         FixedCapacityExponentialHistogram morePositiveValues = createAutoReleasedHistogram(100);
         morePositiveValues.resetBuckets(0);
         morePositiveValues.tryAddBucket(1999, 1, false);
@@ -83,4 +85,85 @@ public class ExponentialHistogramUtilsTests extends ExponentialHistogramTestCase
         );
         assertThat(sum, equalTo(Double.NEGATIVE_INFINITY));
     }
+
+    public void testMinimumEstimation() {
+        for (int i = 0; i < 100; i++) {
+            int positiveValueCount = randomBoolean() ? 0 : randomIntBetween(10, 10_000);
+            int negativeValueCount = randomBoolean() ? 0 : randomIntBetween(10, 10_000);
+            int zeroValueCount = randomBoolean() ? 0 : randomIntBetween(10, 100);
+            int bucketCount = randomIntBetween(4, 500);
+
+            double correctMin = Double.MAX_VALUE;
+            double zeroThreshold = Double.MAX_VALUE;
+            double[] values = new double[positiveValueCount + negativeValueCount];
+            for (int j = 0; j < values.length; j++) {
+                double absValue = Math.pow(10, randomIntBetween(1, 9)) * randomDouble();
+                if (j < positiveValueCount) {
+                    values[j] = absValue;
+                } else {
+                    values[j] = -absValue;
+                }
+                zeroThreshold = Math.min(zeroThreshold, absValue / 2);
+                correctMin = Math.min(correctMin, values[j]);
+            }
+            if (zeroValueCount > 0) {
+                correctMin = Math.min(correctMin, -zeroThreshold);
+            }
+
+            ExponentialHistogram histo = createAutoReleasedHistogram(bucketCount, values);
+
+            OptionalDouble estimatedMin = ExponentialHistogramUtils.estimateMin(
+                new ZeroBucket(zeroThreshold, zeroValueCount),
+                histo.negativeBuckets(),
+                histo.positiveBuckets()
+            );
+            if (correctMin == Double.MAX_VALUE) {
+                assertThat(estimatedMin.isPresent(), equalTo(false));
+            } else {
+                assertThat(estimatedMin.isPresent(), equalTo(true));
+                // If the histogram does not contain mixed sign values, we have a guaranteed relative error bound of 2^(2^-scale) - 1
+                double histogramBase = Math.pow(2, Math.pow(2, -histo.scale()));
+                double allowedError = Math.abs(correctMin * (histogramBase - 1));
+                assertThat(estimatedMin.getAsDouble(), closeTo(correctMin, allowedError));
+            }
+        }
+    }
+
+    public void testMinimumEstimationPositiveInfinityHandling() {
+        FixedCapacityExponentialHistogram histo = createAutoReleasedHistogram(100);
+        histo.resetBuckets(0);
+        histo.tryAddBucket(2000, 1, true);
+
+        OptionalDouble estimate = ExponentialHistogramUtils.estimateMin(
+            ZeroBucket.minimalEmpty(),
+            histo.negativeBuckets(),
+            histo.positiveBuckets()
+        );
+        assertThat(estimate.isPresent(), equalTo(true));
+        assertThat(estimate.getAsDouble(), equalTo(Double.POSITIVE_INFINITY));
+    }
+
+    public void testMinimumEstimationNegativeInfinityHandling() {
+        FixedCapacityExponentialHistogram histo = createAutoReleasedHistogram(100);
+        histo.resetBuckets(0);
+        histo.tryAddBucket(2000, 1, false);
+
+        OptionalDouble estimate = ExponentialHistogramUtils.estimateMin(
+            ZeroBucket.minimalEmpty(),
+            histo.negativeBuckets(),
+            histo.positiveBuckets()
+        );
+        assertThat(estimate.isPresent(), equalTo(true));
+        assertThat(estimate.getAsDouble(), equalTo(Double.NEGATIVE_INFINITY));
+    }
+
+    public void testMinimumEstimationSanitizedNegativeZero() {
+        OptionalDouble estimate = ExponentialHistogramUtils.estimateMin(
+            ZeroBucket.minimalWithCount(42),
+            ExponentialHistogram.empty().negativeBuckets(),
+            ExponentialHistogram.empty().positiveBuckets()
+        );
+        assertThat(estimate.isPresent(), equalTo(true));
+        assertThat(estimate.getAsDouble(), equalTo(0.0));
+    }
 }

+ 8 - 3
libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContentTests.java

@@ -41,6 +41,7 @@ public class ExponentialHistogramXContentTests extends ExponentialHistogramTestC
         histo.setZeroBucket(new ZeroBucket(0.1234, 42));
         histo.resetBuckets(7);
         histo.setSum(1234.56);
+        histo.setMin(-321.123);
         histo.tryAddBucket(-10, 15, false);
         histo.tryAddBucket(10, 5, false);
         histo.tryAddBucket(-11, 10, true);
@@ -51,6 +52,7 @@ public class ExponentialHistogramXContentTests extends ExponentialHistogramTestC
                 "{"
                     + "\"scale\":7,"
                     + "\"sum\":1234.56,"
+                    + "\"min\":-321.123,"
                     + "\"zero\":{\"count\":42,\"threshold\":0.1234},"
                     + "\"positive\":{\"indices\":[-11,11],\"counts\":[10,20]},"
                     + "\"negative\":{\"indices\":[-10,10],\"counts\":[15,5]}"
@@ -72,25 +74,28 @@ public class ExponentialHistogramXContentTests extends ExponentialHistogramTestC
         histo.setZeroBucket(new ZeroBucket(0.0, 7));
         histo.resetBuckets(2);
         histo.setSum(1.1);
-        assertThat(toJson(histo), equalTo("{\"scale\":2,\"sum\":1.1,\"zero\":{\"count\":7}}"));
+        histo.setMin(0);
+        assertThat(toJson(histo), equalTo("{\"scale\":2,\"sum\":1.1,\"min\":0.0,\"zero\":{\"count\":7}}"));
     }
 
     public void testOnlyPositiveBuckets() {
         FixedCapacityExponentialHistogram histo = createAutoReleasedHistogram(10);
         histo.resetBuckets(4);
         histo.setSum(1.1);
+        histo.setMin(0.5);
         histo.tryAddBucket(-1, 3, true);
         histo.tryAddBucket(2, 5, true);
-        assertThat(toJson(histo), equalTo("{\"scale\":4,\"sum\":1.1,\"positive\":{\"indices\":[-1,2],\"counts\":[3,5]}}"));
+        assertThat(toJson(histo), equalTo("{\"scale\":4,\"sum\":1.1,\"min\":0.5,\"positive\":{\"indices\":[-1,2],\"counts\":[3,5]}}"));
     }
 
     public void testOnlyNegativeBuckets() {
         FixedCapacityExponentialHistogram histo = createAutoReleasedHistogram(10);
         histo.resetBuckets(5);
         histo.setSum(1.1);
+        histo.setMin(-0.5);
         histo.tryAddBucket(-1, 4, false);
         histo.tryAddBucket(2, 6, false);
-        assertThat(toJson(histo), equalTo("{\"scale\":5,\"sum\":1.1,\"negative\":{\"indices\":[-1,2],\"counts\":[4,6]}}"));
+        assertThat(toJson(histo), equalTo("{\"scale\":5,\"sum\":1.1,\"min\":-0.5,\"negative\":{\"indices\":[-1,2],\"counts\":[4,6]}}"));
     }
 
     private static String toJson(ExponentialHistogram histo) {

+ 11 - 2
x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/CompressedExponentialHistogram.java

@@ -34,6 +34,7 @@ public class CompressedExponentialHistogram implements ExponentialHistogram {
     private double zeroThreshold;
     private long valueCount;
     private double sum;
+    private double min;
     private ZeroBucket lazyZeroBucket;
 
     private final EncodedHistogramData encodedData = new EncodedHistogramData();
@@ -59,6 +60,11 @@ public class CompressedExponentialHistogram implements ExponentialHistogram {
         return sum;
     }
 
+    @Override
+    public double min() {
+        return min;
+    }
+
     @Override
     public ExponentialHistogram.Buckets positiveBuckets() {
         return positiveBuckets;
@@ -75,14 +81,17 @@ public class CompressedExponentialHistogram implements ExponentialHistogram {
      * @param zeroThreshold the zeroThreshold for the histogram, which needs to be stored externally
      * @param valueCount the total number of values the histogram contains, needs to be stored externally
      * @param sum the total sum of the values the histogram contains, needs to be stored externally
+     * @param min the minimum of the values the histogram contains, needs to be stored externally.
+     *            Must be {@link Double#NaN} if the histogram is empty.
      * @param encodedHistogramData the encoded histogram bytes which previously where generated via
      * {@link #writeHistogramBytes(StreamOutput, int, List, List)}.
      */
-    public void reset(double zeroThreshold, long valueCount, double sum, BytesRef encodedHistogramData) throws IOException {
+    public void reset(double zeroThreshold, long valueCount, double sum, double min, BytesRef encodedHistogramData) throws IOException {
         lazyZeroBucket = null;
         this.zeroThreshold = zeroThreshold;
         this.valueCount = valueCount;
         this.sum = sum;
+        this.min = min;
         encodedData.decode(encodedHistogramData);
         negativeBuckets.resetCachedData();
         positiveBuckets.resetCachedData();
@@ -90,7 +99,7 @@ public class CompressedExponentialHistogram implements ExponentialHistogram {
 
     /**
      * Serializes the given histogram, so that exactly the same data can be reconstructed via
-     * {@link #reset(double, long, double, BytesRef)}.
+     * {@link #reset(double, long, double, double, BytesRef)}.
      *
      * @param output the output to write the serialized bytes to
      * @param scale the scale of the histogram

+ 46 - 3
x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java

@@ -22,6 +22,7 @@ import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramUtils;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent;
+import org.elasticsearch.exponentialhistogram.ZeroBucket;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
@@ -48,6 +49,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.OptionalDouble;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.exponentialhistogram.ExponentialHistogram.MAX_INDEX;
@@ -72,6 +74,8 @@ import static org.elasticsearch.exponentialhistogram.ExponentialHistogram.MIN_IN
  * <pre><code>{
  *   "my_histo": {
  *     "scale": 12,
+ *     "sum": 1234,
+ *     "min": -123.456,
  *     "zero": {
  *       "threshold": 0.123456,
  *       "count": 42
@@ -95,6 +99,7 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
 
     public static final ParseField SCALE_FIELD = new ParseField(ExponentialHistogramXContent.SCALE_FIELD);
     public static final ParseField SUM_FIELD = new ParseField(ExponentialHistogramXContent.SUM_FIELD);
+    public static final ParseField MIN_FIELD = new ParseField(ExponentialHistogramXContent.MIN_FIELD);
     public static final ParseField ZERO_FIELD = new ParseField(ExponentialHistogramXContent.ZERO_FIELD);
     public static final ParseField ZERO_COUNT_FIELD = new ParseField(ExponentialHistogramXContent.ZERO_COUNT_FIELD);
     public static final ParseField ZERO_THRESHOLD_FIELD = new ParseField(ExponentialHistogramXContent.ZERO_THRESHOLD_FIELD);
@@ -140,6 +145,10 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
         return fullPath + "._values_sum";
     }
 
+    private static String valuesMinSubFieldName(String fullPath) {
+        return fullPath + "._values_min";
+    }
+
     static class Builder extends FieldMapper.Builder {
 
         private final FieldMapper.Parameter<Map<String, String>> meta = FieldMapper.Parameter.metaParam();
@@ -266,6 +275,7 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
             }
 
             Double sum = null;
+            Double min = null;
             Integer scale = null;
             ParsedZeroBucket zeroBucket = ParsedZeroBucket.DEFAULT;
             List<IndexWithCount> negativeBuckets = Collections.emptyList();
@@ -305,6 +315,8 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
                     }
                 } else if (fieldName.equals(SUM_FIELD.getPreferredName())) {
                     sum = parseDoubleAllowingInfinity(subParser);
+                } else if (fieldName.equals(MIN_FIELD.getPreferredName())) {
+                    min = parseDoubleAllowingInfinity(subParser);
                 } else if (fieldName.equals(ZERO_FIELD.getPreferredName())) {
                     zeroBucket = parseZeroBucket(subParser);
                 } else if (fieldName.equals(POSITIVE_FIELD.getPreferredName())) {
@@ -347,8 +359,8 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
 
             if (sum == null) {
                 sum = ExponentialHistogramUtils.estimateSum(
-                    IndexWithCount.asBucketIterator(scale, negativeBuckets),
-                    IndexWithCount.asBucketIterator(scale, positiveBuckets)
+                    IndexWithCount.asBuckets(scale, negativeBuckets).iterator(),
+                    IndexWithCount.asBuckets(scale, positiveBuckets).iterator()
                 );
             } else {
                 if (totalValueCount == 0 && sum != 0.0) {
@@ -359,6 +371,22 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
                 }
             }
 
+            if (min == null) {
+                OptionalDouble estimatedMin = ExponentialHistogramUtils.estimateMin(
+                    new ZeroBucket(zeroBucket.threshold(), zeroBucket.count),
+                    IndexWithCount.asBuckets(scale, negativeBuckets),
+                    IndexWithCount.asBuckets(scale, positiveBuckets)
+                );
+                if (estimatedMin.isPresent()) {
+                    min = estimatedMin.getAsDouble();
+                }
+            } else if (totalValueCount == 0) {
+                throw new DocumentParsingException(
+                    subParser.getTokenLocation(),
+                    "error parsing field [" + fullPath() + "], min field must be null if the histogram is empty, but got " + min
+                );
+            }
+
             BytesStreamOutput histogramBytesOutput = new BytesStreamOutput();
             CompressedExponentialHistogram.writeHistogramBytes(histogramBytesOutput, scale, negativeBuckets, positiveBuckets);
             BytesRef histoBytes = histogramBytesOutput.bytes().toBytesRef();
@@ -376,6 +404,13 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
             context.doc().add(zeroThresholdField);
             context.doc().add(valuesCountField);
             context.doc().add(sumField);
+            if (min != null) {
+                NumericDocValuesField minField = new NumericDocValuesField(
+                    valuesMinSubFieldName(fullPath()),
+                    NumericUtils.doubleToSortableLong(min)
+                );
+                context.doc().add(minField);
+            }
 
         } catch (Exception ex) {
             if (ignoreMalformed.value() == false) {
@@ -629,6 +664,7 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
         private double zeroThreshold;
         private long valueCount;
         private double valueSum;
+        private double valueMin;
 
         @Override
         public SourceLoader.SyntheticFieldLoader.DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf)
@@ -642,6 +678,7 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
             NumericDocValues zeroThresholds = leafReader.getNumericDocValues(zeroThresholdSubFieldName(fullPath()));
             NumericDocValues valueCounts = leafReader.getNumericDocValues(valuesCountSubFieldName(fullPath()));
             NumericDocValues valueSums = leafReader.getNumericDocValues(valuesSumSubFieldName(fullPath()));
+            NumericDocValues valueMins = leafReader.getNumericDocValues(valuesMinSubFieldName(fullPath()));
             assert zeroThresholds != null;
             assert valueCounts != null;
             assert valueSums != null;
@@ -657,6 +694,12 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
                     zeroThreshold = NumericUtils.sortableLongToDouble(zeroThresholds.longValue());
                     valueCount = valueCounts.longValue();
                     valueSum = NumericUtils.sortableLongToDouble(valueSums.longValue());
+
+                    if (valueMins != null && valueMins.advanceExact(docId)) {
+                        valueMin = NumericUtils.sortableLongToDouble(valueMins.longValue());
+                    } else {
+                        valueMin = Double.NaN;
+                    }
                     return true;
                 }
                 binaryValue = null;
@@ -675,7 +718,7 @@ public class ExponentialHistogramFieldMapper extends FieldMapper {
                 return;
             }
 
-            histogram.reset(zeroThreshold, valueCount, valueSum, binaryValue);
+            histogram.reset(zeroThreshold, valueCount, valueSum, valueMin, binaryValue);
             ExponentialHistogramXContent.serialize(b, histogram);
         }
 

+ 55 - 19
x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/IndexWithCount.java

@@ -7,9 +7,11 @@
 
 package org.elasticsearch.xpack.exponentialhistogram;
 
-import org.elasticsearch.exponentialhistogram.BucketIterator;
+import org.elasticsearch.exponentialhistogram.CopyableBucketIterator;
+import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
 
 import java.util.List;
+import java.util.OptionalLong;
 
 /**
  * An exponential histogram bucket represented by its index and associated bucket count.
@@ -17,34 +19,68 @@ import java.util.List;
  * @param count the number of values in that bucket.
  */
 public record IndexWithCount(long index, long count) {
-    public static BucketIterator asBucketIterator(int scale, List<IndexWithCount> buckets) {
-        return new BucketIterator() {
-            int position = 0;
 
+    static ExponentialHistogram.Buckets asBuckets(int scale, List<IndexWithCount> bucketIndices) {
+        return new ExponentialHistogram.Buckets() {
             @Override
-            public boolean hasNext() {
-                return position < buckets.size();
+            public CopyableBucketIterator iterator() {
+                return new Iterator(bucketIndices, scale, 0);
             }
 
             @Override
-            public long peekCount() {
-                return buckets.get(position).count;
+            public OptionalLong maxBucketIndex() {
+                if (bucketIndices.isEmpty()) {
+                    return OptionalLong.empty();
+                }
+                return OptionalLong.of(bucketIndices.get(bucketIndices.size() - 1).index);
             }
 
             @Override
-            public long peekIndex() {
-                return buckets.get(position).index;
+            public long valueCount() {
+                throw new UnsupportedOperationException("not implemented");
             }
+        };
+    }
 
-            @Override
-            public void advance() {
-                position++;
-            }
+    private static class Iterator implements CopyableBucketIterator {
+        private final List<IndexWithCount> buckets;
+        private final int scale;
+        private int position;
 
-            @Override
-            public int scale() {
-                return scale;
-            }
-        };
+        Iterator(List<IndexWithCount> buckets, int scale, int position) {
+            this.buckets = buckets;
+            this.scale = scale;
+            this.position = position;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return position < buckets.size();
+        }
+
+        @Override
+        public long peekCount() {
+            return buckets.get(position).count;
+        }
+
+        @Override
+        public long peekIndex() {
+            return buckets.get(position).index;
+        }
+
+        @Override
+        public void advance() {
+            position++;
+        }
+
+        @Override
+        public int scale() {
+            return scale;
+        }
+
+        @Override
+        public CopyableBucketIterator copy() {
+            return new Iterator(buckets, scale, position);
+        }
     }
 }

+ 50 - 6
x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java

@@ -9,6 +9,7 @@ package org.elasticsearch.xpack.exponentialhistogram;
 
 import org.elasticsearch.core.Types;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramUtils;
+import org.elasticsearch.exponentialhistogram.ZeroBucket;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentParsingException;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -30,6 +31,7 @@ import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.OptionalDouble;
 import java.util.Set;
 
 import static org.elasticsearch.exponentialhistogram.ExponentialHistogram.MAX_INDEX;
@@ -129,8 +131,13 @@ public class ExponentialHistogramFieldMapperTests extends MapperTestCase {
                 Map.of("indices", negativeIndices, "counts", negativeCounts)
             )
         );
-        if (randomBoolean() && (positiveIndices.isEmpty() == false || negativeIndices.isEmpty() == false)) {
-            result.put("sum", randomDoubleBetween(-1000, 1000, true));
+        if ((positiveIndices.isEmpty() == false || negativeIndices.isEmpty() == false)) {
+            if (randomBoolean()) {
+                result.put("sum", randomDoubleBetween(-1000, 1000, true));
+            }
+            if (randomBoolean()) {
+                result.put("min", randomDoubleBetween(-1000, 1000, true));
+            }
         }
         return result;
     }
@@ -394,6 +401,11 @@ public class ExponentialHistogramFieldMapperTests extends MapperTestCase {
             // Non-Zero sum for empty histogram
             exampleMalformedValue(b -> b.startObject().field("scale", 0).field("sum", 42.0).endObject()).errorMatches(
                 "sum field must be zero if the histogram is empty, but got 42.0"
+            ),
+
+            // Min provided for empty histogram
+            exampleMalformedValue(b -> b.startObject().field("scale", 0).field("min", 42.0).endObject()).errorMatches(
+                "min field must be null if the histogram is empty, but got 42.0"
             )
         );
     }
@@ -442,20 +454,35 @@ public class ExponentialHistogramFieldMapperTests extends MapperTestCase {
                 List<IndexWithCount> positive = parseBuckets(Types.forciblyCast(histogram.get("positive")));
                 List<IndexWithCount> negative = parseBuckets(Types.forciblyCast(histogram.get("negative")));
 
+                Map<String, Object> zeroBucket = convertZeroBucketToCanonicalForm(Types.forciblyCast(histogram.get("zero")));
+
                 Object sum = histogram.get("sum");
                 if (sum == null) {
                     sum = ExponentialHistogramUtils.estimateSum(
-                        IndexWithCount.asBucketIterator(scale, negative),
-                        IndexWithCount.asBucketIterator(scale, positive)
+                        IndexWithCount.asBuckets(scale, negative).iterator(),
+                        IndexWithCount.asBuckets(scale, positive).iterator()
                     );
                 }
                 result.put("sum", sum);
 
-                Map<String, Object> zeroBucket = convertZeroBucketToCanonicalForm(Types.forciblyCast(histogram.get("zero")));
+                Object min = histogram.get("min");
+                if (min == null) {
+                    OptionalDouble estimatedMin = ExponentialHistogramUtils.estimateMin(
+                        mapToZeroBucket(zeroBucket),
+                        IndexWithCount.asBuckets(scale, negative),
+                        IndexWithCount.asBuckets(scale, positive)
+                    );
+                    if (estimatedMin.isPresent()) {
+                        min = estimatedMin.getAsDouble();
+                    }
+                }
+                if (min != null) {
+                    result.put("min", min);
+                }
+
                 if (zeroBucket != null) {
                     result.put("zero", zeroBucket);
                 }
-
                 if (positive.isEmpty() == false) {
                     result.put("positive", writeBucketsInCanonicalForm(positive));
                 }
@@ -466,6 +493,23 @@ public class ExponentialHistogramFieldMapperTests extends MapperTestCase {
                 return result;
             }
 
+            private ZeroBucket mapToZeroBucket(Map<String, Object> zeroBucket) {
+                if (zeroBucket == null) {
+                    return ZeroBucket.minimalEmpty();
+                }
+                Number threshold = Types.forciblyCast(zeroBucket.get("threshold"));
+                Number count = Types.forciblyCast(zeroBucket.get("count"));
+                if (threshold != null && count != null) {
+                    return new ZeroBucket(threshold.doubleValue(), count.longValue());
+                } else if (threshold != null) {
+                    return new ZeroBucket(threshold.doubleValue(), 0);
+                } else if (count != null) {
+                    return ZeroBucket.minimalWithCount(count.longValue());
+                } else {
+                    return ZeroBucket.minimalEmpty();
+                }
+            }
+
             private List<IndexWithCount> parseBuckets(Map<String, Object> buckets) {
                 if (buckets == null) {
                     return List.of();

+ 17 - 3
x-pack/plugin/mapper-exponential-histogram/src/yamlRestTest/resources/rest-api-spec/test/10_synthetic_source.yml

@@ -29,6 +29,7 @@ setup:
           my_histo:
             scale: 12
             sum: 1234.56
+            min: -345.67
             zero:
               threshold: 0.123456
               count: 42
@@ -47,6 +48,7 @@ setup:
      _source.my_histo:
        scale: 12
        sum: 1234.56
+       min: -345.67
        zero:
          threshold: 0.123456
          count: 42
@@ -68,6 +70,7 @@ setup:
           my_histo:
             scale: -10
             sum: 1234.56
+            min: -345.67
             positive:
               indices: [1,2,3,4,5]
               counts: [6,7,8,9,10]
@@ -83,6 +86,7 @@ setup:
       _source.my_histo:
         scale: -10
         sum: 1234.56
+        min: -345.67
         positive:
           indices: [1,2,3,4,5]
           counts: [6,7,8,9,10]
@@ -101,6 +105,7 @@ setup:
           my_histo:
             scale: 0
             sum: 1234.56
+            min: 345.67
             positive:
               indices: [-100, 10, 20]
               counts: [3, 2, 1]
@@ -113,6 +118,7 @@ setup:
       _source.my_histo:
         scale: 0
         sum: 1234.56
+        min: 345.67
         positive:
           indices: [-100, 10, 20]
           counts: [3, 2, 1]
@@ -128,6 +134,7 @@ setup:
           my_histo:
             scale: 0
             sum: 1234.56
+            min: -345.67
             negative:
               indices: [-100, 10, 20]
               counts: [3, 2, 1]
@@ -140,6 +147,7 @@ setup:
       _source.my_histo:
         scale: 0
         sum: 1234.56
+        min: -345.67
         negative:
           indices: [-100, 10, 20]
           counts: [3, 2, 1]
@@ -206,6 +214,7 @@ setup:
       _source.my_histo:
         scale: 0
         sum: 0.0
+        min: 0.0
         zero:
           count: 101
 
@@ -221,6 +230,7 @@ setup:
           my_histo:
             scale: 38
             sum: 1E300
+            min: -1E300
             zero:
               count: 2305843009213693952 # 2^61 to not cause overflows for the total value count sum
               threshold: 1E-300
@@ -238,6 +248,7 @@ setup:
       _source.my_histo:
         scale: 38
         sum: 1E300
+        min: -1E300
         zero:
           count: 2305843009213693952
           threshold: 1E-300
@@ -249,7 +260,7 @@ setup:
           counts: [2305843009213693952, 1]
 
 ---
-"Sum estimation":
+"Aggregates estimation":
   - do:
       index:
         index: test_exponential_histogram
@@ -273,6 +284,7 @@ setup:
       _source.my_histo:
         scale: 1
         sum: 9.289321881345247
+        min: -2.0
         positive:
           indices: [8]
           counts: [1]
@@ -281,7 +293,7 @@ setup:
           counts: [1, 5]
 
 ---
-"Positive infinity sum":
+"Positive infinity aggregates":
   - do:
       index:
         index: test_exponential_histogram
@@ -302,12 +314,13 @@ setup:
       _source.my_histo:
         scale: 0
         sum: Infinity
+        min: Infinity
         positive:
           indices: [2000]
           counts: [1]
 
 ---
-"negative infinity sum":
+"negative infinity aggregates":
   - do:
       index:
         index: test_exponential_histogram
@@ -328,6 +341,7 @@ setup:
       _source.my_histo:
         scale: 0
         sum: -Infinity
+        min: -Infinity
         negative:
           indices: [2000]
           counts: [1]