Convert metric aggs docs runtime fields (#71260)

This replaces the `script` docs for bucket aggregations with runtime
fields. We expect runtime fields to be nicer to work with because you
can also fetch them or filter on them. We expect them to be faster
because their don't need this sort of `instanceof` tree:
https://github.com/elastic/elasticsearch/blob/a92a647b9f17d1bddf5c707490a19482c273eda3/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java#L42

Relates to #69291

Co-authored-by: James Rodewig <40268737+jrodewig@users.noreply.github.com>
Co-authored-by: Adam Locke <adam.locke@elastic.co>

docs/build.gradle

@@ -224,15 +224,16 @@ buildRestTests.setups['messages'] = '''
           refresh: true
           body: |
             {"index":{"_id": "0"}}
-            {"message": "trying out Elasticsearch"}
+            {"message": "trying out Elasticsearch", "context": "foo"}
             {"index":{"_id": "1"}}
-            {"message": "some message with the number 1"}
+            {"message": "some message with the number 1", "context": "bar"}
             {"index":{"_id": "2"}}
-            {"message": "some message with the number 2"}
+            {"message": "some message with the number 2", "context": "bar"}
             {"index":{"_id": "3"}}
-            {"message": "some message with the number 3"}
+            {"message": "some message with the number 3", "context": "bar"}
             {"index":{"_id": "4"}}
-            {"message": "some message with the number 4"}'''
+            {"message": "some message with the number 4", "context": "bar"}
+'''
 
 // Used for EQL
 setups['sec_logs'] = setups['my_data_stream'] + '''
@@ -533,14 +534,6 @@ buildRestTests.setups['exams'] = '''
             {"index":{}}
             {"grade": 50, "weight": 3}'''
 
-buildRestTests.setups['stored_example_script'] = '''
-  # Simple script to load a field. Not really a good example, but a simple one.
-  - do:
-      put_script:
-        id: "my_script"
-        body: { "script": { "lang": "painless", "source": "doc[params.field].value" } }
-  - match: { acknowledged: true }
-'''
 
 buildRestTests.setups['stored_scripted_metric_script'] = '''
   - do:

docs/reference/aggregations/metrics/avg-aggregation.asciidoc

@@ -4,7 +4,7 @@
 <titleabbrev>Avg</titleabbrev>
 ++++
 
-A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script.
+A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents.
 
 Assuming the data consists of documents representing exams grades (between 0
 and 100) of students we can average their scores with:
@@ -39,72 +39,48 @@ The name of the aggregation (`avg_grade` above) also serves as the key by which
 
 ==== Script
 
-Computing the average grade based on a script:
+Let's say the exam was exceedingly difficult, and you need to apply a grade correction. Average a <<runtime,runtime field>> to get a corrected average:
 
 [source,console]
---------------------------------------------------
+----
 POST /exams/_search?size=0
 {
-  "aggs": {
-    "avg_grade": {
-      "avg": {
-        "script": {
-          "source": "doc.grade.value"
+  "runtime_mappings": {
+    "grade.corrected": {
+      "type": "double",
+      "script": {
+        "source": "emit(Math.min(100, doc['grade'].value * params.correction))",
+        "params": {
+          "correction": 1.2
         }
       }
     }
-  }
-}
---------------------------------------------------
-// TEST[setup:exams]
-
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax:
-
-[source,console]
---------------------------------------------------
-POST /exams/_search?size=0
-{
+  },
   "aggs": {
-    "avg_grade": {
+    "avg_corrected_grade": {
       "avg": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "grade"
-          }
-        }
+        "field": "grade.corrected"
       }
     }
   }
 }
---------------------------------------------------
-// TEST[setup:exams,stored_example_script]
-
-===== Value Script
-
-It turned out that the exam was way above the level of the students and a grade correction needs to be applied. We can use value script to get the new average:
+----
+// TEST[setup:exams]
+// TEST[s/size=0/size=0&filter_path=aggregations/]
 
-[source,console]
---------------------------------------------------
-POST /exams/_search?size=0
+////
+[source,console-result]
+----
 {
-  "aggs": {
+  "aggregations": {
     "avg_corrected_grade": {
-      "avg": {
-        "field": "grade",
-        "script": {
-          "lang": "painless",
-          "source": "_value * params.correction",
-          "params": {
-            "correction": 1.2
-          }
-        }
-      }
+      "value": 80.0
     }
   }
 }
---------------------------------------------------
-// TEST[setup:exams]
+----
+////
+
 
 ==== Missing value
 

docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc

@@ -7,8 +7,7 @@
 ++++
 
 A `boxplot` metrics aggregation that computes boxplot of numeric values extracted from the aggregated documents.
-These values can be generated by a provided script or extracted from specific numeric or
-<<histogram,histogram fields>> in the documents.
+These values can be generated from specific numeric or <<histogram,histogram fields>> in the documents.
 
 The `boxplot` aggregation returns essential information for making a {wikipedia}/Box_plot[box plot]: minimum, maximum,
 median, first quartile (25th percentile)  and third quartile (75th percentile) values.
@@ -76,59 +75,56 @@ query for them directly.
 
 ==== Script
 
-The boxplot metric supports scripting. For example, if our load times
-are in milliseconds but we want values calculated in seconds, we could use
-a script to convert them on-the-fly:
+If you need to create a boxplot for values that aren't indexed exactly you
+should create a <<runtime,runtime field>> and get the boxplot of that. For
+example, if your load times are in milliseconds but you want values calculated
+in seconds, use a runtime field to convert them:
 
 [source,console]
---------------------------------------------------
+----
 GET latency/_search
 {
   "size": 0,
-  "aggs": {
-    "load_time_boxplot": {
-      "boxplot": {
-        "script": {
-          "lang": "painless",
-          "source": "doc['load_time'].value / params.timeUnit", <1>
-          "params": {
-            "timeUnit": 1000                                    <2>
-          }
+  "runtime_mappings": {
+    "load_time.seconds": {
+      "type": "long",
+      "script": {
+        "source": "emit(doc['load_time'].value / params.timeUnit)",
+        "params": {
+          "timeUnit": 1000
         }
       }
     }
+  },
+  "aggs": {
+    "load_time_boxplot": {
+      "boxplot": { "field": "load_time.seconds" }
+    }
   }
 }
---------------------------------------------------
+----
 // TEST[setup:latency]
+// TEST[s/_search/_search?filter_path=aggregations/]
+// TEST[s/"timeUnit": 1000/"timeUnit": 10/]
 
-<1> The `field` parameter is replaced with a `script` parameter, which uses the
-script to generate values which percentiles are calculated on
-<2> Scripting supports parameterized input just like any other script
-
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a
-stored script use the following syntax:
-
-[source,console]
+////
+[source,console-result]
 --------------------------------------------------
-GET latency/_search
 {
-  "size": 0,
-  "aggs": {
+ "aggregations": {
     "load_time_boxplot": {
-      "boxplot": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "load_time"
-          }
-        }
-      }
+      "min": 0.0,
+      "max": 99.0,
+      "q1": 16.5,
+      "q2": 44.5,
+      "q3": 72.5,
+      "lower": 0.0,
+      "upper": 99.0
     }
   }
 }
 --------------------------------------------------
-// TEST[setup:latency,stored_example_script]
+////
 
 [[search-aggregations-metrics-boxplot-aggregation-approximation]]
 ==== Boxplot values are (usually) approximate

docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc

@@ -5,8 +5,7 @@
 ++++
 
 A `single-value` metrics aggregation that calculates an approximate count of
-distinct values. Values can be extracted either from specific fields in the
-document or generated by a script.
+distinct values.
 
 Assume you are indexing store sales and would like to count the unique number of sold products that match a query:
 
@@ -181,49 +180,43 @@ make sure that hashes are computed at most once per unique value per segment.
 
 ==== Script
 
-The `cardinality` metric supports scripting, with a noticeable performance hit
-however since hashes need to be computed on the fly.
+If you need the cardinality of the combination of two fields,
+create a <<runtime,runtime field>> combining them and aggregate it.
 
 [source,console]
---------------------------------------------------
+----
 POST /sales/_search?size=0
 {
+  "runtime_mappings": {
+    "type_and_promoted": {
+      "type": "keyword",
+      "script": "emit(doc['type'].value + ' ' + doc['promoted'].value)"
+    }
+  },
   "aggs": {
     "type_promoted_count": {
       "cardinality": {
-        "script": {
-          "lang": "painless",
-          "source": "doc['type'].value + ' ' + doc['promoted'].value"
-        }
+        "field": "type_and_promoted"
       }
     }
   }
 }
---------------------------------------------------
+----
 // TEST[setup:sales]
+// TEST[s/size=0/size=0&filter_path=aggregations/]
 
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax:
-
-[source,console]
+////
+[source,console-result]
 --------------------------------------------------
-POST /sales/_search?size=0
 {
-  "aggs": {
+  "aggregations": {
     "type_promoted_count": {
-      "cardinality": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "type_field": "type",
-            "promoted_field": "promoted"
-          }
-        }
-      }
+      "value": 5
     }
   }
 }
 --------------------------------------------------
-// TEST[skip:no script]
+////
 
 ==== Missing value
 

docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc

@@ -4,7 +4,7 @@
 <titleabbrev>Extended stats</titleabbrev>
 ++++
 
-A `multi-value` metrics aggregation that computes stats over numeric values extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script.
+A `multi-value` metrics aggregation that computes stats over numeric values extracted from the aggregated documents.
 
 The `extended_stats` aggregations is an extended version of the <<search-aggregations-metrics-stats-aggregation,`stats`>> aggregation, where additional metrics are added such as `sum_of_squares`, `variance`, `std_deviation` and `std_deviation_bounds`.
 
@@ -100,76 +100,67 @@ if your data is skewed heavily left or right, the value returned will be mislead
 
 ==== Script
 
-Computing the grades stats based on a script:
+If you need to aggregate on a value that isn't indexed, use a <<runtime,runtime field>>.
+Say the we found out that the grades we've been working on were for an exam that was above
+the level of the students and we want to "correct" it:
 
 [source,console]
---------------------------------------------------
+----
 GET /exams/_search
 {
   "size": 0,
-  "aggs": {
-    "grades_stats": {
-      "extended_stats": {
-        "script": {
-          "source": "doc['grade'].value",
-          "lang": "painless"
+  "runtime_mappings": {
+    "grade.corrected": {
+      "type": "double",
+      "script": {
+        "source": "emit(Math.min(100, doc['grade'].value * params.correction))",
+        "params": {
+          "correction": 1.2
         }
       }
     }
-  }
-}
---------------------------------------------------
-// TEST[setup:exams]
-
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax:
-
-[source,console]
---------------------------------------------------
-GET /exams/_search
-{
-  "size": 0,
+  },
   "aggs": {
     "grades_stats": {
-      "extended_stats": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "grade"
-          }
-        }
-      }
+      "extended_stats": { "field": "grade.corrected" }
     }
   }
 }
---------------------------------------------------
-// TEST[setup:exams,stored_example_script]
-
-===== Value Script
-
-It turned out that the exam was way above the level of the students and a grade correction needs to be applied. We can use value script to get the new stats:
+----
+// TEST[setup:exams]
+// TEST[s/_search/_search?filter_path=aggregations/]
 
-[source,console]
---------------------------------------------------
-GET /exams/_search
+////
+[source,console-result]
+----
 {
-  "size": 0,
-  "aggs": {
+  "aggregations": {
     "grades_stats": {
-      "extended_stats": {
-        "field": "grade",
-        "script": {
-          "lang": "painless",
-          "source": "_value * params.correction",
-          "params": {
-            "correction": 1.2
-          }
-        }
+      "count": 2,
+      "min": 60.0,
+      "max": 100.0,
+      "avg": 80.0,
+      "sum": 160.0,
+      "sum_of_squares": 13600.0,
+      "variance": 400.0,
+      "variance_population": 400.0,
+      "variance_sampling": 800.0,
+      "std_deviation": 20.0,
+      "std_deviation_population": 20.0,
+      "std_deviation_sampling": 28.284271247461902,
+      "std_deviation_bounds": {
+        "upper": 120.0,
+        "lower": 40.0,
+        "upper_population": 120.0,
+        "lower_population": 40.0,
+        "upper_sampling": 136.5685424949238,
+        "lower_sampling": 23.431457505076196
       }
     }
   }
 }
---------------------------------------------------
-// TEST[setup:exams]
+----
+////
 
 ==== Missing value
 

docs/reference/aggregations/metrics/matrix-stats-aggregation.asciidoc

@@ -140,7 +140,3 @@ GET /_search
 --------------------------------------------------
 
 <1> Documents without a value in the `income` field will have the default value `50000`.
-
-==== Script
-
-This aggregation family does not yet support scripting.

docs/reference/aggregations/metrics/max-aggregation.asciidoc

@@ -5,9 +5,7 @@
 ++++
 
 A `single-value` metrics aggregation that keeps track and returns the maximum
-value among the numeric values extracted from the aggregated documents. These
-values can be extracted either from specific numeric fields in the documents,
-or be generated by a provided script.
+value among the numeric values extracted from the aggregated documents.
 
 NOTE: The `min` and `max` aggregation operate on the `double` representation of
 the data. As a consequence, the result may be approximate when running on longs
@@ -47,76 +45,49 @@ response.
 
 ==== Script
 
-The `max` aggregation can also calculate the maximum of a script. The example
-below computes the maximum price:
+If you need to get the `max` of something more complex than a single field,
+run an aggregation on a <<runtime,runtime field>>.
 
 [source,console]
---------------------------------------------------
+----
 POST /sales/_search
 {
-  "aggs" : {
-      "max_price" : {
-          "max" : {
-              "script" : {
-                  "source" : "doc.price.value"
-              }
-          }
-      }
+  "size": 0,
+  "runtime_mappings": {
+    "price.adjusted": {
+      "type": "double",
+      "script": """
+        double price = doc['price'].value;
+        if (doc['promoted'].value) {
+          price *= 0.8;
+        }
+        emit(price);
+      """
+    }
+  },
+  "aggs": {
+    "max_price": {
+      "max": { "field": "price.adjusted" }
+    }
   }
 }
---------------------------------------------------
+----
 // TEST[setup:sales]
+// TEST[s/_search/_search?filter_path=aggregations/]
 
-This will use the <<modules-scripting-painless, Painless>> scripting language
-and no script parameters. To use a stored script use the following syntax:
-
-[source,console]
+////
+[source,console-result]
 --------------------------------------------------
-POST /sales/_search
 {
-  "aggs" : {
-      "max_price" : {
-          "max" : {
-              "script" : {
-                  "id": "my_script",
-                  "params": {
-                      "field": "price"
-                  }
-              }
-          }
+  "aggregations": {
+      "max_price": {
+          "value": 175.0
       }
   }
 }
 --------------------------------------------------
-// TEST[setup:sales,stored_example_script]
-
-==== Value Script
-
-Let's say that the prices of the documents in our index are in USD, but we
-would like to compute the max in EURO (and for the sake of this example, let's
-say the conversion rate is 1.2). We can use a value script to apply the
-conversion rate to every value before it is aggregated:
+////
 
-[source,console]
---------------------------------------------------
-POST /sales/_search
-{
-  "aggs" : {
-      "max_price_in_euros" : {
-          "max" : {
-              "field" : "price",
-              "script" : {
-                  "source" : "_value * params.conversion_rate",
-                  "params" : {
-                      "conversion_rate" : 1.2
-                  }
-              }
-          }
-      }
-  }
-}
---------------------------------------------------
-// TEST[setup:sales]
 
 ==== Missing value
 

docs/reference/aggregations/metrics/median-absolute-deviation-aggregation.asciidoc

@@ -109,56 +109,56 @@ but observed performance will depend on the sample data.
 
 ==== Script
 
-This metric aggregation supports scripting. In our example above, product
-reviews are on a scale of one to five. If we wanted to modify them to a scale
-of one to ten, we can using scripting.
-
-To provide an inline script:
+In the example above, product reviews are on a scale of one to five. If you
+want to modify them to a scale of one to ten, use a <<runtime,runtime field>>.
 
 [source,console]
----------------------------------------------------------
-GET reviews/_search
+----
+GET reviews/_search?filter_path=aggregations
 {
   "size": 0,
+  "runtime_mappings": {
+    "rating.out_of_ten": {
+      "type": "long",
+      "script": {
+        "source": "emit(doc['rating'].value * params.scaleFactor)",
+        "params": {
+          "scaleFactor": 2
+        }
+      }
+    }
+  },
   "aggs": {
+    "review_average": {
+      "avg": {
+        "field": "rating.out_of_ten"
+      }
+    },
     "review_variability": {
       "median_absolute_deviation": {
-        "script": {
-          "lang": "painless",
-          "source": "doc['rating'].value * params.scaleFactor",
-          "params": {
-            "scaleFactor": 2
-          }
-        }
+        "field": "rating.out_of_ten"
       }
     }
   }
 }
----------------------------------------------------------
+----
 // TEST[setup:reviews]
 
-To provide a stored script:
+Which will result in:
 
-[source,console]
+[source,console-result]
 ---------------------------------------------------------
-GET reviews/_search
 {
-  "size": 0,
-  "aggs": {
+  "aggregations": {
+    "review_average": {
+      "value": 6.0
+    },
     "review_variability": {
-      "median_absolute_deviation": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "rating"
-          }
-        }
-      }
+      "value": 4.0
     }
   }
 }
 ---------------------------------------------------------
-// TEST[setup:reviews,stored_example_script]
 
 ==== Missing value
 

docs/reference/aggregations/metrics/min-aggregation.asciidoc

@@ -5,9 +5,7 @@
 ++++
 
 A `single-value` metrics aggregation that keeps track and returns the minimum
-value among numeric values extracted from the aggregated documents. These
-values can be extracted either from specific numeric fields in the documents,
-or be generated by a provided script.
+value among numeric values extracted from the aggregated documents.
 
 NOTE: The `min` and `max` aggregation operate on the `double` representation of
 the data. As a consequence, the result may be approximate when running on longs
@@ -48,76 +46,48 @@ response.
 
 ==== Script
 
-The `min` aggregation can also calculate the minimum of a script. The example
-below computes the minimum price:
+If you need to get the `min` of something more complex than a single field,
+run the aggregation on a <<runtime,runtime field>>.
 
 [source,console]
---------------------------------------------------
+----
 POST /sales/_search
 {
-  "aggs": {
-    "min_price": {
-      "min": {
-        "script": {
-          "source": "doc.price.value"
+  "size": 0,
+  "runtime_mappings": {
+    "price.adjusted": {
+      "type": "double",
+      "script": """
+        double price = doc['price'].value;
+        if (doc['promoted'].value) {
+          price *= 0.8;
         }
-      }
+        emit(price);
+      """
     }
-  }
-}
---------------------------------------------------
-// TEST[setup:sales]
-
-This will use the <<modules-scripting-painless, Painless>> scripting language
-and no script parameters. To use a stored script use the following syntax:
-
-[source,console]
---------------------------------------------------
-POST /sales/_search
-{
+  },
   "aggs": {
     "min_price": {
-      "min": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "price"
-          }
-        }
-      }
+      "min": { "field": "price.adjusted" }
     }
   }
 }
---------------------------------------------------
-// TEST[setup:sales,stored_example_script]
-
-==== Value Script
-
-Let's say that the prices of the documents in our index are in USD, but we
-would like to compute the min in EURO (and for the sake of this example, let's
-say the conversion rate is 1.2). We can use a value script to apply the
-conversion rate to every value before it is aggregated:
+----
+// TEST[setup:sales]
+// TEST[s/_search/_search?filter_path=aggregations/]
 
-[source,console]
+////
+[source,console-result]
 --------------------------------------------------
-POST /sales/_search
 {
-  "aggs": {
-    "min_price_in_euros": {
-      "min": {
-        "field": "price",
-        "script": {
-          "source": "_value * params.conversion_rate",
-          "params": {
-            "conversion_rate": 1.2
-          }
-        }
+  "aggregations": {
+      "min_price": {
+          "value": 8.0
       }
-    }
   }
 }
 --------------------------------------------------
-// TEST[setup:sales]
+////
 
 ==== Missing value
 

docs/reference/aggregations/metrics/percentile-aggregation.asciidoc

@@ -6,8 +6,7 @@
 
 A `multi-value` metrics aggregation that calculates one or more percentiles
 over numeric values extracted from the aggregated documents. These values can be
-generated by a provided script or extracted from specific numeric or
-<<histogram,histogram fields>> in the documents.
+extracted from specific numeric or <<histogram,histogram fields>> in the documents.
 
 Percentiles show the point at which a certain percentage of observed values
 occur. For example, the 95th percentile is the value which is greater than 95%
@@ -164,58 +163,59 @@ Response:
 
 ==== Script
 
-The percentile metric supports scripting. For example, if our load times
-are in milliseconds but we want percentiles calculated in seconds, we could use
-a script to convert them on-the-fly:
+If you need to run the aggregation against values that aren't indexed, use
+a <<runtime,runtime field>>. For example, if our load times
+are in milliseconds but you want percentiles calculated in seconds:
 
 [source,console]
---------------------------------------------------
+----
 GET latency/_search
 {
   "size": 0,
+  "runtime_mappings": {
+    "load_time.seconds": {
+      "type": "long",
+      "script": {
+        "source": "emit(doc['load_time'].value / params.timeUnit)",
+        "params": {
+          "timeUnit": 1000
+        }
+      }
+    }
+  },
   "aggs": {
     "load_time_outlier": {
       "percentiles": {
-        "script": {
-          "lang": "painless",
-          "source": "doc['load_time'].value / params.timeUnit", <1>
-          "params": {
-            "timeUnit": 1000                                    <2>
-          }
-        }
+        "field": "load_time.seconds"
       }
     }
   }
 }
---------------------------------------------------
+----
 // TEST[setup:latency]
+// TEST[s/_search/_search?filter_path=aggregations/]
+// TEST[s/"timeUnit": 1000/"timeUnit": 10/]
 
-<1> The `field` parameter is replaced with a `script` parameter, which uses the
-script to generate values which percentiles are calculated on
-<2> Scripting supports parameterized input just like any other script
-
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax:
-
-[source,console]
---------------------------------------------------
-GET latency/_search
+////
+[source,console-result]
+----
 {
-  "size": 0,
-  "aggs": {
+ "aggregations": {
     "load_time_outlier": {
-      "percentiles": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "load_time"
-          }
-        }
+      "values": {
+        "1.0": 0.5,
+        "5.0": 2.5,
+        "25.0": 16.5,
+        "50.0": 44.5,
+        "75.0": 72.5,
+        "95.0": 94.5,
+        "99.0": 98.5
       }
     }
   }
 }
---------------------------------------------------
-// TEST[setup:latency,stored_example_script]
+----
+////
 
 [[search-aggregations-metrics-percentile-aggregation-approximation]]
 ==== Percentiles are (usually) approximate

docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc

@@ -6,8 +6,7 @@
 
 A `multi-value` metrics aggregation that calculates one or more percentile ranks
 over numeric values extracted from the aggregated documents. These values can be
-generated by a provided script or extracted from specific numeric or
-<<histogram,histogram fields>> in the documents.
+extracted from specific numeric or <<histogram,histogram fields>> in the documents.
 
 [NOTE]
 ==================================================
@@ -120,60 +119,54 @@ Response:
 
 ==== Script
 
-The percentile rank metric supports scripting. For example, if our load times
-are in milliseconds but we want to specify values in seconds, we could use
-a script to convert them on-the-fly:
+If you need to run the aggregation against values that aren't indexed, use
+a <<runtime,runtime field>>. For example, if our load times
+are in milliseconds but we want percentiles calculated in seconds:
 
 [source,console]
---------------------------------------------------
+----
 GET latency/_search
 {
   "size": 0,
+  "runtime_mappings": {
+    "load_time.seconds": {
+      "type": "long",
+      "script": {
+        "source": "emit(doc['load_time'].value / params.timeUnit)",
+        "params": {
+          "timeUnit": 1000
+        }
+      }
+    }
+  },
   "aggs": {
     "load_time_ranks": {
       "percentile_ranks": {
         "values": [ 500, 600 ],
-        "script": {
-          "lang": "painless",
-          "source": "doc['load_time'].value / params.timeUnit", <1>
-          "params": {
-            "timeUnit": 1000                                    <2>
-          }
-        }
+        "field": "load_time.seconds"
       }
     }
   }
 }
---------------------------------------------------
+----
 // TEST[setup:latency]
+// TEST[s/_search/_search?filter_path=aggregations/]
 
-<1> The `field` parameter is replaced with a `script` parameter, which uses the
-script to generate values which percentile ranks are calculated on
-<2> Scripting supports parameterized input just like any other script
-
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax:
-
-[source,console]
+////
+[source,console-result]
 --------------------------------------------------
-GET latency/_search
 {
-  "size": 0,
-  "aggs": {
+  "aggregations": {
     "load_time_ranks": {
-      "percentile_ranks": {
-        "values": [ 500, 600 ],
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "load_time"
-          }
-        }
+      "values": {
+        "500.0": 100.0,
+        "600.0": 100.0
       }
     }
   }
 }
 --------------------------------------------------
-// TEST[setup:latency,stored_example_script]
+////
 
 ==== HDR Histogram
 

docs/reference/aggregations/metrics/rate-aggregation.asciidoc

@@ -7,7 +7,7 @@
 ++++
 
 A `rate` metrics aggregation can be used only inside a `date_histogram` and calculates a rate of documents or a field in each
-`date_histogram` bucket. The field values can be generated by a provided script or extracted from specific numeric or
+`date_histogram` bucket. The field values can be generated extracted from specific numeric or
 <<histogram,histogram fields>> in the documents.
 
 ==== Syntax
@@ -244,8 +244,6 @@ By default `sum` mode is used.
 `"mode": "sum"`:: calculate the sum of all values field
 `"mode": "value_count"`:: use the number of values in the field
 
-The `mode` parameter can only be used with fields and scripts.
-
 ==== Relationship between bucket sizes and rate
 
 The `rate` aggregation supports all rate that can be used <<calendar_intervals,calendar_intervals parameter>> of `date_histogram`
@@ -268,14 +266,26 @@ is `day` based, only  `second`, ` minute`, `hour`, `day`, and `week` rate interv
 
 ==== Script
 
-The `rate` aggregation also supports scripting. For example, if we need to adjust out prices before calculating rates, we could use
-a script to recalculate them on-the-fly:
+If you need to run the aggregation against values that aren't indexed, run the
+aggregation on a <<runtime,runtime field>>. For example, if we need to adjust
+our prices before calculating rates:
 
 [source,console]
---------------------------------------------------
+----
 GET sales/_search
 {
   "size": 0,
+  "runtime_mappings": {
+    "price.adjusted": {
+      "type": "double",
+      "script": {
+        "source": "emit(doc['price'].value * params.adjustment)",
+        "params": {
+          "adjustment": 0.9
+        }
+      }
+    }
+  },
   "aggs": {
     "by_date": {
       "date_histogram": {
@@ -285,28 +295,18 @@ GET sales/_search
       "aggs": {
         "avg_price": {
           "rate": {
-            "script": {  <1>
-              "lang": "painless",
-              "source": "doc['price'].value * params.adjustment",
-              "params": {
-                "adjustment": 0.9  <2>
-              }
-            }
+            "field": "price.adjusted"
           }
         }
       }
     }
   }
 }
---------------------------------------------------
+----
 // TEST[setup:sales]
 
-<1> The `field` parameter is replaced with a `script` parameter, which uses the
-script to generate values which percentiles are calculated on.
-<2> Scripting supports parameterized input just like any other script.
-
 [source,console-result]
---------------------------------------------------
+----
 {
   ...
   "aggregations" : {
@@ -340,5 +340,5 @@ script to generate values which percentiles are calculated on.
     }
   }
 }
---------------------------------------------------
+----
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]

docs/reference/aggregations/metrics/stats-aggregation.asciidoc

@@ -4,7 +4,7 @@
 <titleabbrev>Stats</titleabbrev>
 ++++
 
-A `multi-value` metrics aggregation that computes stats over numeric values extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script.
+A `multi-value` metrics aggregation that computes stats over numeric values extracted from the aggregated documents.
 
 The stats that are returned consist of: `min`, `max`, `sum`, `count` and `avg`.
 
@@ -46,73 +46,51 @@ The name of the aggregation (`grades_stats` above) also serves as the key by whi
 
 ==== Script
 
-Computing the grades stats based on a script:
+If you need to get the `stats` for something more complex than a single field,
+run the aggregation on a <<runtime,runtime field>>.
 
 [source,console]
 --------------------------------------------------
-POST /exams/_search?size=0
+POST /exams/_search
 {
+  "size": 0,
+  "runtime_mappings": {
+    "grade.weighted": {
+      "type": "double",
+      "script": """
+        emit(doc['grade'].value * doc['weight'].value)
+      """
+    }
+  },
   "aggs": {
     "grades_stats": {
       "stats": {
-        "script": {
-          "lang": "painless",
-          "source": "doc['grade'].value"
-        }
+        "field": "grade.weighted"
       }
     }
   }
 }
 --------------------------------------------------
 // TEST[setup:exams]
+// TEST[s/_search/_search?filter_path=aggregations/]
 
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax:
-
-[source,console]
+////
+[source,console-result]
 --------------------------------------------------
-POST /exams/_search?size=0
 {
-  "aggs": {
+  "aggregations": {
     "grades_stats": {
-      "stats": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "grade"
-          }
-        }
-      }
+      "count": 2,
+      "min": 150.0,
+      "max": 200.0,
+      "avg": 175.0,
+      "sum": 350.0
     }
   }
 }
 --------------------------------------------------
-// TEST[setup:exams,stored_example_script]
+////
 
-===== Value Script
-
-It turned out that the exam was way above the level of the students and a grade correction needs to be applied. We can use a value script to get the new stats:
-
-[source,console]
---------------------------------------------------
-POST /exams/_search?size=0
-{
-  "aggs": {
-    "grades_stats": {
-      "stats": {
-        "field": "grade",
-        "script": {
-          "lang": "painless",
-          "source": "_value * params.correction",
-          "params": {
-            "correction": 1.2
-          }
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-// TEST[setup:exams]
 
 ==== Missing value
 

docs/reference/aggregations/metrics/string-stats-aggregation.asciidoc

@@ -7,10 +7,7 @@
 ++++
 
 A `multi-value` metrics aggregation that computes statistics over string values extracted from the aggregated documents.
-These values can be retrieved either from specific `keyword` fields in the documents or can be generated by a provided script.
-
-WARNING: Using scripts can result in slower search speeds. See
-<<scripts-and-search-speed>>.
+These values can be retrieved either from specific `keyword` fields.
 
 The string stats aggregation returns the following results:
 
@@ -130,74 +127,48 @@ The `distribution` object shows the probability of each character appearing in a
 
 ==== Script
 
-Computing the message string stats based on a script:
+If you need to get the `string_stats` for something more complex than a single
+field, run the aggregation on a <<runtime,runtime field>>.
 
 [source,console]
---------------------------------------------------
-POST /my-index-000001/_search?size=0
+----
+POST /my-index-000001/_search
 {
-  "aggs": {
-    "message_stats": {
-      "string_stats": {
-        "script": {
-          "lang": "painless",
-          "source": "doc['message.keyword'].value"
-        }
-      }
+  "size": 0,
+  "runtime_mappings": {
+    "message_and_context": {
+      "type": "keyword",
+      "script": """
+        emit(doc['message.keyword'].value + ' ' + doc['context.keyword'].value)
+      """
     }
-  }
-}
---------------------------------------------------
-// TEST[setup:messages]
-
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters.
-To use a stored script use the following syntax:
-
-[source,console]
---------------------------------------------------
-POST /my-index-000001/_search?size=0
-{
+  },
   "aggs": {
     "message_stats": {
-      "string_stats": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "message.keyword"
-          }
-        }
-      }
+      "string_stats": { "field": "message_and_context" }
     }
   }
 }
---------------------------------------------------
-// TEST[setup:messages,stored_example_script]
-
-===== Value Script
-
-We can use a value script to modify the message (eg we can add a prefix) and compute the new stats:
+----
+// TEST[setup:messages]
+// TEST[s/_search/_search?filter_path=aggregations/]
 
-[source,console]
---------------------------------------------------
-POST /my-index-000001/_search?size=0
+////
+[source,console-result]
+----
 {
-  "aggs": {
+  "aggregations": {
     "message_stats": {
-      "string_stats": {
-        "field": "message.keyword",
-        "script": {
-          "lang": "painless",
-          "source": "params.prefix + _value",
-          "params": {
-            "prefix": "Message: "
-          }
-        }
-      }
+      "count": 5,
+      "min_length": 28,
+      "max_length": 34,
+      "avg_length": 32.8,
+      "entropy": 3.9797778402765784
     }
   }
 }
---------------------------------------------------
-// TEST[setup:messages]
+----
+////
 
 ==== Missing value
 

docs/reference/aggregations/metrics/sum-aggregation.asciidoc

@@ -5,8 +5,7 @@
 ++++
 
 A `single-value` metrics aggregation that sums up numeric values that are extracted from the aggregated documents.
-These values can be extracted either from specific numeric or <<histogram,histogram>> fields in the documents,
-or be generated by a provided script.
+These values can be extracted either from specific numeric or <<histogram,histogram>> fields.
 
 Assuming the data consists of documents representing sales records we can sum
 the sale price of all hats with:
@@ -48,38 +47,25 @@ The name of the aggregation (`hat_prices` above) also serves as the key by which
 
 ==== Script
 
-We could also use a script to fetch the sales price:
+If you need to get the `sum` for something more complex than a single
+field, run the aggregation on a <<runtime,runtime field>>.
 
 [source,console]
---------------------------------------------------
+----
 POST /sales/_search?size=0
 {
-  "query": {
-    "constant_score": {
-      "filter": {
-        "match": { "type": "hat" }
-      }
-    }
-  },
-  "aggs": {
-    "hat_prices": {
-      "sum": {
-        "script": {
-          "source": "doc.price.value"
+  "runtime_mappings": {
+    "price.weighted": {
+      "type": "double",
+      "script": """
+        double price = doc['price'].value;
+        if (doc['promoted'].value) {
+          price *= 0.8;
         }
-      }
+        emit(price);
+      """
     }
-  }
-}
---------------------------------------------------
-// TEST[setup:sales]
-
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax:
-
-[source,console]
---------------------------------------------------
-POST /sales/_search?size=0
-{
+  },
   "query": {
     "constant_score": {
       "filter": {
@@ -90,48 +76,27 @@ POST /sales/_search?size=0
   "aggs": {
     "hat_prices": {
       "sum": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "price"
-          }
-        }
+        "field": "price.weighted"
       }
     }
   }
 }
---------------------------------------------------
-// TEST[setup:sales,stored_example_script]
-
-===== Value Script
-
-It is also possible to access the field value from the script using `_value`.
-For example, this will sum the square of the prices for all hats:
+----
+// TEST[setup:sales]
+// TEST[s/size=0/size=0&filter_path=aggregations/]
 
-[source,console]
---------------------------------------------------
-POST /sales/_search?size=0
+////
+[source,console-result]
+----
 {
-  "query": {
-    "constant_score": {
-      "filter": {
-        "match": { "type": "hat" }
-      }
-    }
-  },
-  "aggs": {
-    "square_hats": {
-      "sum": {
-        "field": "price",
-        "script": {
-          "source": "_value * _value"
-        }
-      }
+  "aggregations": {
+    "hat_prices": {
+      "value": 370.0
     }
   }
 }
---------------------------------------------------
-// TEST[setup:sales]
+----
+////
 
 ==== Missing value
 

docs/reference/aggregations/metrics/t-test-aggregation.asciidoc

@@ -7,7 +7,7 @@
 ++++
 
 A `t_test` metrics aggregation that performs a statistical hypothesis test in which the test statistic follows a Student's t-distribution
-under the null hypothesis on numeric values extracted from the aggregated documents or generated by provided scripts. In practice, this
+under the null hypothesis on numeric values extracted from the aggregated documents. In practice, this
 will tell you if the difference between two population means are statistically significant and did not occur by chance alone.
 
 ==== Syntax
@@ -136,45 +136,58 @@ GET node_upgrade/_search
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 <1> The p-value.
 
-In this example, we are using the same fields for both populations. However this is not a requirement and different fields and even
-combination of fields and scripts can be used. Populations don't have to be in the same index either. If data sets are located in different
+Populations don't have to be in the same index. If data sets are located in different
 indices, the term filter on the <<mapping-index-field,`_index`>> field can be used to select populations.
 
 ==== Script
 
-The `t_test` metric supports scripting. For example, if we need to adjust out load times for the before values, we could use
-a script to recalculate them on-the-fly:
+If you need to run the `t_test` on values that aren't represented cleanly
+by a field you should, run the aggregation on a <<runtime,runtime field>>.
+For example, if you want to adjust out load times for the before values:
 
 [source,console]
---------------------------------------------------
+----
 GET node_upgrade/_search
 {
   "size": 0,
+  "runtime_mappings": {
+    "startup_time_before.adjusted": {
+      "type": "long",
+      "script": {
+        "source": "emit(doc['startup_time_before'].value - params.adjustment)",
+        "params": {
+          "adjustment": 10
+        }
+      }
+    }
+  },
   "aggs": {
     "startup_time_ttest": {
       "t_test": {
         "a": {
-          "script": {
-            "lang": "painless",
-            "source": "doc['startup_time_before'].value - params.adjustment", <1>
-            "params": {
-              "adjustment": 10                                                <2>
-            }
-          }
+          "field": "startup_time_before.adjusted"
         },
         "b": {
-          "field": "startup_time_after"                                       <3>
+          "field": "startup_time_after"
         },
         "type": "paired"
       }
     }
   }
 }
---------------------------------------------------
+----
 // TEST[setup:node_upgrade]
+// TEST[s/_search/_search?filter_path=aggregations/]
 
-<1> The `field` parameter is replaced with a `script` parameter, which uses the
-script to generate values which percentiles are calculated on.
-<2> Scripting supports parameterized input just like any other script.
-<3> We can mix scripts and fields.
-
+////
+[source,console-result]
+----
+{
+ "aggregations": {
+    "startup_time_ttest": {
+      "value": 0.9397399375119482
+    }
+  }
+}
+----
+////

docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc

@@ -9,8 +9,7 @@ These values can be extracted either from specific fields in the documents, or b
 this aggregator will be used in conjunction with other single-value aggregations. For example, when computing the `avg`
 one might be interested in the number of values the average is computed over.
 
-`value_count` does not de-duplicate values, so even if a field has duplicates (or a script generates multiple
-identical values for a single document), each value will be counted individually.
+`value_count` does not de-duplicate values, so even if a field has duplicates each value will be counted individually.
 
 [source,console]
 --------------------------------------------------
@@ -43,50 +42,49 @@ retrieved from the returned response.
 
 ==== Script
 
-Counting the values generated by a script:
+If you need to count something more complex than the values in a single field
+you should run the aggregation on a <<runtime,runtime field>>.
 
 [source,console]
---------------------------------------------------
-POST /sales/_search?size=0
+----
+POST /sales/_search
 {
+  "size": 0,
+  "runtime_mappings": {
+    "tags": {
+      "type": "keyword",
+      "script": """
+        emit(doc['type'].value);
+        if (doc['promoted'].value) {
+          emit('hot');
+        }
+      """
+    }
+  },
   "aggs": {
-    "type_count": {
+    "tags_count": {
       "value_count": {
-        "script": {
-          "source": "doc['type'].value"
-        }
+        "field": "tags"
       }
     }
   }
 }
---------------------------------------------------
+----
 // TEST[setup:sales]
+// TEST[s/_search/_search?filter_path=aggregations/]
 
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax:
-
-[source,console]
---------------------------------------------------
-POST /sales/_search?size=0
+////
+[source,console-result]
+----
 {
-  "aggs": {
-    "types_count": {
-      "value_count": {
-        "script": {
-          "id": "my_script",
-          "params": {
-            "field": "type"
-          }
-        }
-      }
+  "aggregations": {
+    "tags_count": {
+      "value": 12
     }
   }
 }
---------------------------------------------------
-// TEST[setup:sales,stored_example_script]
-
-NOTE:: Because `value_count` is designed to work with any field it internally treats all values as simple bytes.
-Due to this implementation, if `_value` script variable is used to fetch a value instead of accessing the field
-directly (e.g. a "value script"), the field value will be returned as a string instead of it's native format.
+----
+////
 
 [[search-aggregations-metrics-valuecount-aggregation-histogram-fields]]
 ==== Histogram fields

docs/reference/aggregations/metrics/weighted-avg-aggregation.asciidoc

@@ -5,11 +5,11 @@
 ++++
 
 A `single-value` metrics aggregation that computes the weighted average of numeric values that are extracted from the aggregated documents.
-These values can be extracted either from specific numeric fields in the documents, or provided by a script.
+These values can be extracted either from specific numeric fields in the documents.
 
 When calculating a regular average, each datapoint has an equal "weight" ... it contributes equally to the final value. Weighted averages,
 on the other hand, weight each datapoint differently. The amount that each datapoint contributes to the final value is extracted from the
-document, or provided by a script.
+document.
 
 As a formula, a weighted average is the `∑(value * weight) / ∑(weight)`
 
@@ -23,7 +23,6 @@ A regular average can be thought of as a weighted average where every value has
 |`value` | The configuration for the field or script that provides the values |Required |
 |`weight` | The configuration for the field or script that provides the weights |Required |
 |`format` | The numeric response formatter |Optional |
-|`value_type` | A hint about the values for pure scripts or unmapped fields |Optional |
 |===
 
 The `value` and `weight` objects have per-field specific configuration:
@@ -35,7 +34,6 @@ The `value` and `weight` objects have per-field specific configuration:
 |Parameter Name |Description |Required |Default Value
 |`field` | The field that values should be extracted from |Required |
 |`missing` | A value to use if the field is missing entirely |Optional |
-|`script` | A script which provides the values for the document. This is mutually exclusive with `field` |Optional
 |===
 
 [[weight-params]]
@@ -45,7 +43,6 @@ The `value` and `weight` objects have per-field specific configuration:
 |Parameter Name |Description |Required |Default Value
 |`field` | The field that weights should be extracted from |Required |
 |`missing` | A weight to use if the field is missing entirely |Optional |
-|`script` | A script which provides the weights for the document. This is mutually exclusive with `field` |Optional
 |===
 
 
@@ -92,9 +89,9 @@ Which yields a response like:
 
 
 While multiple values-per-field are allowed, only one weight is allowed. If the aggregation encounters
-a document that has more than one weight (e.g. the weight field is a multi-valued field) it will throw an exception.
-If you have this situation, you will need to specify a `script` for the weight field, and use the script
-to combine the multiple values into a single value to be used.
+a document that has more than one weight (e.g. the weight field is a multi-valued field) it will abort the search.
+If you have this situation, you should build a <<search-aggregations-metrics-weight-avg-aggregation-runtime-field>>
+to combine those values into a single weight.
 
 This single weight will be applied independently to each value extracted from the `value` field.
 
@@ -145,16 +142,40 @@ The three values (`1`, `2`, and `3`) will be included as independent values, all
 The aggregation returns `2.0` as the result, which matches what we would expect when calculating by hand:
 `((1*2) + (2*2) + (3*2)) / (2+2+2) == 2`
 
-==== Script
+[[search-aggregations-metrics-weight-avg-aggregation-runtime-field]]
+==== Runtime field
 
-Both the value and the weight can be derived from a script, instead of a field. As a simple example, the following
-will add one to the grade and weight in the document using a script:
+If you have to sum or weigh values that don't quite line up with the indexed
+values, run the aggregation on a <<runtime,runtime field>>.
 
 [source,console]
---------------------------------------------------
-POST /exams/_search
+----
+POST /exams/_doc?refresh
+{
+  "grade": 100,
+  "weight": [2, 3]
+}
+POST /exams/_doc?refresh
+{
+  "grade": 80,
+  "weight": 3
+}
+
+POST /exams/_search?filter_path=aggregations
 {
   "size": 0,
+  "runtime_mappings": {
+    "weight.combined": {
+      "type": "double",
+      "script": """
+        double s = 0;
+        for (double w : doc['weight']) {
+          s += w;
+        }
+        emit(s);
+      """
+    }
+  },
   "aggs": {
     "weighted_grade": {
       "weighted_avg": {
@@ -162,14 +183,26 @@ POST /exams/_search
           "script": "doc.grade.value + 1"
         },
         "weight": {
-          "script": "doc.weight.value + 1"
+          "field": "weight.combined"
         }
       }
     }
   }
 }
---------------------------------------------------
-// TEST[setup:exams]
+----
+
+Which should look like:
+
+[source,console-result]
+----
+{
+  "aggregations": {
+    "weighted_grade": {
+      "value": 93.5
+    }
+  }
+}
+----
 
 
 ==== Missing values
@@ -204,4 +237,3 @@ POST /exams/_search
 }
 --------------------------------------------------
 // TEST[setup:exams]
-

docs/reference/search/search-your-data/highlighting.asciidoc

@@ -864,7 +864,8 @@ Response:
         "_id": "1",
         "_score": 1.6011951,
         "_source": {
-          "message": "some message with the number 1"
+          "message": "some message with the number 1",
+          "context": "bar"
         },
         "highlight": {
           "message": [
@@ -918,7 +919,8 @@ Response:
         "_id": "1",
         "_score": 1.6011951,
         "_source": {
-          "message": "some message with the number 1"
+          "message": "some message with the number 1",
+          "context": "bar"
         },
         "highlight": {
           "message": [