Upgrade t-digest to 3.2 (#28295) (#28305)

docs/reference/aggregations/metrics/percentile-aggregation.asciidoc

@@ -53,13 +53,13 @@ percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`.  The response will look like this:
    "aggregations": {
       "load_time_outlier": {
          "values" : {
-            "1.0": 9.9,
-            "5.0": 29.500000000000004,
-            "25.0": 167.5,
+            "1.0": 5.0,
+            "5.0": 25.0,
+            "25.0": 165.0,
             "50.0": 445.0,
-            "75.0": 722.5,
-            "95.0": 940.5,
-            "99.0": 980.1000000000001
+            "75.0": 725.0,
+            "95.0": 945.0,
+            "99.0": 985.0
          }
       }
    }
@@ -129,15 +129,15 @@ Response:
             "values": [
                 {
                     "key": 1.0,
-                    "value": 9.9
+                    "value": 5.0
                 },
                 {
                     "key": 5.0,
-                    "value": 29.500000000000004
+                    "value": 25.0
                 },
                 {
                     "key": 25.0,
-                    "value": 167.5
+                    "value": 165.0
                 },
                 {
                     "key": 50.0,
@@ -145,15 +145,15 @@ Response:
                 },
                 {
                     "key": 75.0,
-                    "value": 722.5
+                    "value": 725.0
                 },
                 {
                     "key": 95.0,
-                    "value": 940.5
+                    "value": 945.0
                 },
                 {
                     "key": 99.0,
-                    "value": 980.1000000000001
+                    "value": 985.0
                 }
             ]
         }

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml

@@ -65,21 +65,21 @@ setup:
   - match:  { hits.total: 4 }
   - length: { hits.hits: 4 }
 
-  - match:  { aggregations.percentiles_int.values.1\.0: 2.5 }
-  - match:  { aggregations.percentiles_int.values.5\.0: 8.500000000000002 }
-  - match:  { aggregations.percentiles_int.values.25\.0: 38.5 }
+  - match:  { aggregations.percentiles_int.values.1\.0: 1.0 }
+  - match:  { aggregations.percentiles_int.values.5\.0: 1.0 }
+  - match:  { aggregations.percentiles_int.values.25\.0: 26.0 }
   - match:  { aggregations.percentiles_int.values.50\.0: 76.0 }
-  - match:  { aggregations.percentiles_int.values.75\.0: 113.5 }
-  - match:  { aggregations.percentiles_int.values.95\.0: 143.49999999999997 }
-  - match:  { aggregations.percentiles_int.values.99\.0: 149.5 }
+  - match:  { aggregations.percentiles_int.values.75\.0: 126.0 }
+  - match:  { aggregations.percentiles_int.values.95\.0: 151.0 }
+  - match:  { aggregations.percentiles_int.values.99\.0: 151.0 }
 
-  - match:  { aggregations.percentiles_double.values.1\.0: 2.5 }
-  - match:  { aggregations.percentiles_double.values.5\.0: 8.500000000000002 }
-  - match:  { aggregations.percentiles_double.values.25\.0: 38.5 }
+  - match:  { aggregations.percentiles_double.values.1\.0: 1.0 }
+  - match:  { aggregations.percentiles_double.values.5\.0: 1.0 }
+  - match:  { aggregations.percentiles_double.values.25\.0: 26.0 }
   - match:  { aggregations.percentiles_double.values.50\.0: 76.0 }
-  - match:  { aggregations.percentiles_double.values.75\.0: 113.5 }
-  - match:  { aggregations.percentiles_double.values.95\.0: 143.49999999999997 }
-  - match:  { aggregations.percentiles_double.values.99\.0: 149.5 }
+  - match:  { aggregations.percentiles_double.values.75\.0: 126.0 }
+  - match:  { aggregations.percentiles_double.values.95\.0: 151.0 }
+  - match:  { aggregations.percentiles_double.values.99\.0: 151.0 }
 
   - do:
       search:
@@ -100,21 +100,21 @@ setup:
   - match: { hits.total: 4 }
   - length: { hits.hits: 4 }
 
-  - match:  { aggregations.percentiles_int.values.1\.0: 2.5 }
-  - match:  { aggregations.percentiles_int.values.5\.0: 8.500000000000002 }
-  - match:  { aggregations.percentiles_int.values.25\.0: 38.5 }
+  - match:  { aggregations.percentiles_int.values.1\.0: 1.0 }
+  - match:  { aggregations.percentiles_int.values.5\.0: 1.0 }
+  - match:  { aggregations.percentiles_int.values.25\.0: 26.0 }
   - match:  { aggregations.percentiles_int.values.50\.0: 76.0 }
-  - match:  { aggregations.percentiles_int.values.75\.0: 113.5 }
-  - match:  { aggregations.percentiles_int.values.95\.0: 143.49999999999997 }
-  - match:  { aggregations.percentiles_int.values.99\.0: 149.5 }
+  - match:  { aggregations.percentiles_int.values.75\.0: 126.0 }
+  - match:  { aggregations.percentiles_int.values.95\.0: 151.0 }
+  - match:  { aggregations.percentiles_int.values.99\.0: 151.0 }
 
-  - match:  { aggregations.percentiles_double.values.1\.0: 2.5 }
-  - match:  { aggregations.percentiles_double.values.5\.0: 8.500000000000002 }
-  - match:  { aggregations.percentiles_double.values.25\.0: 38.5 }
+  - match:  { aggregations.percentiles_double.values.1\.0: 1.0 }
+  - match:  { aggregations.percentiles_double.values.5\.0: 1.0 }
+  - match:  { aggregations.percentiles_double.values.25\.0: 26.0 }
   - match:  { aggregations.percentiles_double.values.50\.0: 76.0 }
-  - match:  { aggregations.percentiles_double.values.75\.0: 113.5 }
-  - match:  { aggregations.percentiles_double.values.95\.0: 143.49999999999997 }
-  - match:  { aggregations.percentiles_double.values.99\.0: 149.5 }
+  - match:  { aggregations.percentiles_double.values.75\.0: 126.0 }
+  - match:  { aggregations.percentiles_double.values.95\.0: 151.0 }
+  - match:  { aggregations.percentiles_double.values.99\.0: 151.0 }
 
 
 ---
@@ -135,21 +135,21 @@ setup:
   - match: { hits.total: 4 }
   - length: { hits.hits: 0 }
 
-  - match:  { aggregations.percentiles_int.values.1\.0: 2.5 }
-  - match:  { aggregations.percentiles_int.values.5\.0: 8.500000000000002 }
-  - match:  { aggregations.percentiles_int.values.25\.0: 38.5 }
+  - match:  { aggregations.percentiles_int.values.1\.0: 1.0 }
+  - match:  { aggregations.percentiles_int.values.5\.0: 1.0 }
+  - match:  { aggregations.percentiles_int.values.25\.0: 26.0 }
   - match:  { aggregations.percentiles_int.values.50\.0: 76.0 }
-  - match:  { aggregations.percentiles_int.values.75\.0: 113.5 }
-  - match:  { aggregations.percentiles_int.values.95\.0: 143.49999999999997 }
-  - match:  { aggregations.percentiles_int.values.99\.0: 149.5 }
+  - match:  { aggregations.percentiles_int.values.75\.0: 126.0 }
+  - match:  { aggregations.percentiles_int.values.95\.0: 151.0 }
+  - match:  { aggregations.percentiles_int.values.99\.0: 151.0 }
 
-  - match:  { aggregations.percentiles_double.values.1\.0: 2.5 }
-  - match:  { aggregations.percentiles_double.values.5\.0: 8.500000000000002 }
-  - match:  { aggregations.percentiles_double.values.25\.0: 38.5 }
+  - match:  { aggregations.percentiles_double.values.1\.0: 1.0 }
+  - match:  { aggregations.percentiles_double.values.5\.0: 1.0 }
+  - match:  { aggregations.percentiles_double.values.25\.0: 26.0 }
   - match:  { aggregations.percentiles_double.values.50\.0: 76.0 }
-  - match:  { aggregations.percentiles_double.values.75\.0: 113.5 }
-  - match:  { aggregations.percentiles_double.values.95\.0: 143.49999999999997 }
-  - match:  { aggregations.percentiles_double.values.99\.0: 149.5 }
+  - match:  { aggregations.percentiles_double.values.75\.0: 126.0 }
+  - match:  { aggregations.percentiles_double.values.95\.0: 151.0 }
+  - match:  { aggregations.percentiles_double.values.99\.0: 151.0 }
 
 
 
@@ -176,21 +176,21 @@ setup:
   - match: { hits.total: 3 }
   - length: { hits.hits: 3 }
 
-  - match:  { aggregations.percentiles_int.values.1\.0: 52.0 }
-  - match:  { aggregations.percentiles_int.values.5\.0: 56.0 }
-  - match:  { aggregations.percentiles_int.values.25\.0: 76.0 }
+  - match:  { aggregations.percentiles_int.values.1\.0: 51.0 }
+  - match:  { aggregations.percentiles_int.values.5\.0: 51.0 }
+  - match:  { aggregations.percentiles_int.values.25\.0: 63.5 }
   - match:  { aggregations.percentiles_int.values.50\.0: 101.0 }
-  - match:  { aggregations.percentiles_int.values.75\.0: 126.0 }
-  - match:  { aggregations.percentiles_int.values.95\.0: 146.0 }
-  - match:  { aggregations.percentiles_int.values.99\.0: 150.0 }
+  - match:  { aggregations.percentiles_int.values.75\.0: 138.5 }
+  - match:  { aggregations.percentiles_int.values.95\.0: 151.0 }
+  - match:  { aggregations.percentiles_int.values.99\.0: 151.0 }
 
-  - match:  { aggregations.percentiles_double.values.1\.0: 52.0 }
-  - match:  { aggregations.percentiles_double.values.5\.0: 56.0 }
-  - match:  { aggregations.percentiles_double.values.25\.0: 76.0 }
+  - match:  { aggregations.percentiles_double.values.1\.0: 51.0 }
+  - match:  { aggregations.percentiles_double.values.5\.0: 51.0 }
+  - match:  { aggregations.percentiles_double.values.25\.0: 63.5 }
   - match:  { aggregations.percentiles_double.values.50\.0: 101.0 }
-  - match:  { aggregations.percentiles_double.values.75\.0: 126.0 }
-  - match:  { aggregations.percentiles_double.values.95\.0: 146.0 }
-  - match:  { aggregations.percentiles_double.values.99\.0: 150.0 }
+  - match:  { aggregations.percentiles_double.values.75\.0: 138.5 }
+  - match:  { aggregations.percentiles_double.values.95\.0: 151.0 }
+  - match:  { aggregations.percentiles_double.values.99\.0: 151.0 }
 
 ---
 "Missing field with missing param":
@@ -248,13 +248,13 @@ setup:
   - match: { aggregations.percentiles_int.meta.foo: "bar" }
 
 
-  - match:  { aggregations.percentiles_int.values.1\.0: 2.5 }
-  - match:  { aggregations.percentiles_int.values.5\.0: 8.500000000000002 }
-  - match:  { aggregations.percentiles_int.values.25\.0: 38.5 }
+  - match:  { aggregations.percentiles_int.values.1\.0: 1.0 }
+  - match:  { aggregations.percentiles_int.values.5\.0: 1.0 }
+  - match:  { aggregations.percentiles_int.values.25\.0: 26.0 }
   - match:  { aggregations.percentiles_int.values.50\.0: 76.0 }
-  - match:  { aggregations.percentiles_int.values.75\.0: 113.5 }
-  - match:  { aggregations.percentiles_int.values.95\.0: 143.49999999999997 }
-  - match:  { aggregations.percentiles_int.values.99\.0: 149.5 }
+  - match:  { aggregations.percentiles_int.values.75\.0: 126.0 }
+  - match:  { aggregations.percentiles_int.values.95\.0: 151.0 }
+  - match:  { aggregations.percentiles_int.values.99\.0: 151.0 }
 
 ---
 "Invalid params test":
@@ -329,12 +329,12 @@ setup:
   - match: { hits.total: 4 }
   - length: { hits.hits: 4 }
 
-  - match:  { aggregations.percentiles_int.values.5\.0: 8.500000000000002 }
-  - match:  { aggregations.percentiles_int.values.25\.0: 38.5 }
-  - match:  { aggregations.percentiles_int.values.50\.0: 76.0 }
+  - match: { aggregations.percentiles_int.values.5\.0: 1.0 }
+  - match: { aggregations.percentiles_int.values.25\.0: 26.0 }
+  - match: { aggregations.percentiles_int.values.50\.0: 76.0 }
 
-  - match:  { aggregations.percentiles_double.values.5\.0: 8.500000000000002 }
-  - match: { aggregations.percentiles_double.values.25\.0: 38.5 }
+  - match: { aggregations.percentiles_double.values.5\.0: 1.0 }
+  - match: { aggregations.percentiles_double.values.25\.0: 26.0 }
   - match: { aggregations.percentiles_double.values.50\.0: 76.0 }
 
 ---
@@ -355,9 +355,9 @@ setup:
   - length: { hits.hits: 4 }
 
   - match:  { aggregations.percentiles_int.values.0.key:  5.0 }
-  - match:  { aggregations.percentiles_int.values.0.value:  8.500000000000002 }
+  - match:  { aggregations.percentiles_int.values.0.value:  1.0 }
   - match:  { aggregations.percentiles_int.values.1.key:  25.0 }
-  - match:  { aggregations.percentiles_int.values.1.value:  38.5 }
+  - match:  { aggregations.percentiles_int.values.1.value:  26.0 }
   - match:  { aggregations.percentiles_int.values.2.key:  50.0 }
   - match:  { aggregations.percentiles_int.values.2.value:  76.0 }
 

server/build.gradle

@@ -99,7 +99,7 @@ dependencies {
   compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
 
   // percentiles aggregation
-  compile 'com.tdunning:t-digest:3.0'
+  compile 'com.tdunning:t-digest:3.2'
   // precentil ranks aggregation
   compile 'org.hdrhistogram:HdrHistogram:2.1.9'
 

server/licenses/t-digest-3.0.jar.sha1

@@ -1 +0,0 @@
-84ccf145ac2215e6bfa63baa3101c0af41017cfc

server/licenses/t-digest-3.2.jar.sha1

@@ -0,0 +1 @@
+2ab94758b0276a8a26102adf8d528cf6d0567b9a

server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java

@@ -69,7 +69,6 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
     }
 
     private static double[] randomPercents(long minValue, long maxValue) {
-
         final int length = randomIntBetween(1, 20);
         final double[] percents = new double[length];
         for (int i = 0; i < percents.length; ++i) {
@@ -97,7 +96,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         return builder;
     }
 
-    private void assertConsistent(double[] pcts, PercentileRanks values, long minValue, long maxValue) {
+    private void assertConsistent(double[] pcts, PercentileRanks values, long minValue) {
         final List<Percentile> percentileList = CollectionUtils.iterableAsArrayList(values);
         assertEquals(pcts.length, percentileList.size());
         for (int i = 0; i < pcts.length; ++i) {
@@ -109,9 +108,6 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
             if (percentile.getPercent() == 0) {
                 assertThat(percentile.getValue(), lessThanOrEqualTo((double) minValue));
             }
-            if (percentile.getPercent() == 100) {
-                assertThat(percentile.getValue(), greaterThanOrEqualTo((double) maxValue));
-            }
         }
 
         for (int i = 1; i < percentileList.size(); ++i) {
@@ -193,7 +189,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValue, maxValue);
+        assertConsistent(pcts, values, minValue);
     }
 
     @Override
@@ -233,7 +229,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValue, maxValue);
+        assertConsistent(pcts, values, minValue);
     }
 
     @Override
@@ -248,7 +244,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValue, maxValue);
+        assertConsistent(pcts, values, minValue);
     }
 
     @Override
@@ -266,7 +262,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValue - 1, maxValue - 1);
+        assertConsistent(pcts, values, minValue - 1);
     }
 
     @Override
@@ -286,7 +282,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValue - 1, maxValue - 1);
+        assertConsistent(pcts, values, minValue - 1);
     }
 
     @Override
@@ -301,7 +297,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValues, maxValues);
+        assertConsistent(pcts, values, minValues);
     }
 
     @Override
@@ -319,7 +315,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValues - 1, maxValues - 1);
+        assertConsistent(pcts, values, minValues - 1);
     }
 
     public void testMultiValuedFieldWithValueScriptReverse() throws Exception {
@@ -336,7 +332,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, -maxValues, -minValues);
+        assertConsistent(pcts, values, -maxValues);
     }
 
     @Override
@@ -356,7 +352,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValues - 1, maxValues - 1);
+        assertConsistent(pcts, values, minValues - 1);
     }
 
     @Override
@@ -373,7 +369,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValue, maxValue);
+        assertConsistent(pcts, values, minValue);
     }
 
     @Override
@@ -394,7 +390,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValue - 1, maxValue - 1);
+        assertConsistent(pcts, values, minValue - 1);
     }
 
     @Override
@@ -412,7 +408,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValues, maxValues);
+        assertConsistent(pcts, values, minValues);
     }
 
     @Override
@@ -431,7 +427,7 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
         assertHitCount(searchResponse, 10);
 
         final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks");
-        assertConsistent(pcts, values, minValues - 1, maxValues - 1);
+        assertConsistent(pcts, values, minValues - 1);
     }
 
     public void testOrderBySubAggregation() {

server/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorTests.java

@@ -74,12 +74,12 @@ public class TDigestPercentilesAggregatorTests extends AggregatorTestCase {
         }, tdigest -> {
             assertEquals(7L, tdigest.state.size());
             assertEquals(7L, tdigest.state.centroidCount());
-            assertEquals(4.0d, tdigest.percentile(75), 0.0d);
-            assertEquals("4.0", tdigest.percentileAsString(75));
+            assertEquals(4.5d, tdigest.percentile(75), 0.0d);
+            assertEquals("4.5", tdigest.percentileAsString(75));
             assertEquals(2.0d, tdigest.percentile(50), 0.0d);
             assertEquals("2.0", tdigest.percentileAsString(50));
-            assertEquals(1.0d, tdigest.percentile(20), 0.0d);
-            assertEquals("1.0", tdigest.percentileAsString(20));
+            assertEquals(1.0d, tdigest.percentile(22), 0.0d);
+            assertEquals("1.0", tdigest.percentileAsString(22));
         });
     }
 
@@ -97,14 +97,14 @@ public class TDigestPercentilesAggregatorTests extends AggregatorTestCase {
             assertEquals(tdigest.state.centroidCount(), 7L);
             assertEquals(8.0d, tdigest.percentile(100), 0.0d);
             assertEquals("8.0", tdigest.percentileAsString(100));
-            assertEquals(5.48d, tdigest.percentile(86), 0.0d);
-            assertEquals("5.48", tdigest.percentileAsString(86));
+            assertEquals(6.98d, tdigest.percentile(88), 0.0d);
+            assertEquals("6.98", tdigest.percentileAsString(88));
             assertEquals(1.0d, tdigest.percentile(33), 0.0d);
             assertEquals("1.0", tdigest.percentileAsString(33));
             assertEquals(1.0d, tdigest.percentile(25), 0.0d);
             assertEquals("1.0", tdigest.percentileAsString(25));
-            assertEquals(0.06d, tdigest.percentile(1), 0.0d);
-            assertEquals("0.06", tdigest.percentileAsString(1));
+            assertEquals(0.0d, tdigest.percentile(1), 0.0d);
+            assertEquals("0.0", tdigest.percentileAsString(1));
         });
     }
 
@@ -124,7 +124,7 @@ public class TDigestPercentilesAggregatorTests extends AggregatorTestCase {
             assertEquals(4L, tdigest.state.centroidCount());
             assertEquals(2.0d, tdigest.percentile(100), 0.0d);
             assertEquals(1.0d, tdigest.percentile(50), 0.0d);
-            assertEquals(0.75d, tdigest.percentile(25), 0.0d);
+            assertEquals(0.5d, tdigest.percentile(25), 0.0d);
         });
 
         testCase(LongPoint.newRangeQuery("row", 100, 110), docs, tdigest -> {