ESQL: INLINESTATS docs (#134480)

Fixes https://github.com/elastic/elasticsearch/issues/124718

docs/reference/query-languages/esql/_snippets/commands/examples/inlinestats.csv-spec/avg-salaries-where.md

@@ -0,0 +1,17 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+```esql
+FROM employees
+| KEEP emp_no, salary
+| INLINESTATS avg_lt_50 = ROUND(AVG(salary)) WHERE salary < 50000,
+              avg_lt_60 = ROUND(AVG(salary)) WHERE salary >=50000 AND salary < 60000,
+              avg_gt_60 = ROUND(AVG(salary)) WHERE salary >= 60000
+```
+
+| emp_no:integer | salary:integer | avg_lt_50:double | avg_lt_60:double | avg_gt_60:double |
+| --- | --- | --- | --- | --- |
+| 10001 | 57305 | 38292.0 | 54221.0 | 67286.0 |
+| 10002 | 56371 | 38292.0 | 54221.0 | 67286.0 |
+| 10003 | 61805 | 38292.0 | 54221.0 | 67286.0 |
+| 10004 | 36174 | 38292.0 | 54221.0 | 67286.0 |
+| 10005 | 63528 | 38292.0 | 54221.0 | 67286.0 |

docs/reference/query-languages/esql/_snippets/commands/examples/inlinestats.csv-spec/max-salary-without-by.md

@@ -0,0 +1,15 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+```esql
+FROM employees
+| KEEP emp_no, languages, salary
+| INLINESTATS max_salary = MAX(salary)
+```
+
+| emp_no:integer | languages:integer | salary:integer | max_salary:integer |
+| --- | --- | --- | --- |
+| 10001 | 2 | 57305 | 74999 |
+| 10002 | 5 | 56371 | 74999 |
+| 10003 | 4 | 61805 | 74999 |
+| 10004 | 5 | 36174 | 74999 |
+| 10005 | 1 | 63528 | 74999 |

docs/reference/query-languages/esql/_snippets/commands/examples/inlinestats.csv-spec/max-salary.md

@@ -0,0 +1,15 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+```esql
+FROM employees
+| KEEP emp_no, languages, salary
+| INLINESTATS max_salary = MAX(salary) BY languages
+```
+
+| emp_no:integer | salary:integer | max_salary:integer | languages:integer |
+| --- | --- | --- | --- |
+| 10001 | 57305 | 73578 | 2 |
+| 10002 | 56371 | 66817 | 5 |
+| 10003 | 61805 | 74572 | 4 |
+| 10004 | 36174 | 66817 | 5 |
+| 10005 | 63528 | 73717 | 1 |

docs/reference/query-languages/esql/_snippets/commands/examples/inlinestats.csv-spec/multi-agg-multi-grouping.md

@@ -0,0 +1,18 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+```esql
+FROM employees
+| WHERE still_hired
+| KEEP emp_no, languages, salary, hire_date
+| EVAL tenure = DATE_DIFF("year", hire_date, now())
+| DROP hire_date
+| INLINESTATS avg_salary = AVG(salary), count = count(*) BY languages, tenure
+```
+
+| emp_no:integer | salary:integer | avg_salary:double | count:long | languages:integer | tenure:integer |
+| --- | --- | --- | --- | --- | --- |
+| 10001 | 57305 | 51130.5 | 2 | 2 | 39 |
+| 10002 | 56371 | 40180.0 | 3 | 5 | 39 |
+| 10004 | 36174 | 30749.0 | 2 | 5 | 38 |
+| 10005 | 63528 | 63528.0 | 1 | 1 | 36 |
+| 10007 | 74572 | 58644.0 | 2 | 4 | 36 |

docs/reference/query-languages/esql/_snippets/commands/layout/inlinestats-by.md

@@ -0,0 +1,107 @@
+```yaml {applies_to}
+serverless: preview
+stack: preview 9.2.0
+```
+
+The `INLINE STATS` processing command groups rows according to a common value
+and calculates one or more aggregated values over the grouped rows. The results
+are appended as new columns to the input rows.
+
+The command is identical to [`STATS`](/reference/query-languages/esql/commands/stats-by.md) except that it preserves all the columns from the input table.
+
+**Syntax**
+
+```esql
+INLINE STATS [column1 =] expression1 [WHERE boolean_expression1][,
+      ...,
+      [columnN =] expressionN [WHERE boolean_expressionN]]
+      [BY [grouping_name1 =] grouping_expression1[,
+          ...,
+          [grouping_nameN = ] grouping_expressionN]]
+```
+
+**Parameters**
+
+`columnX`
+:   The name by which the aggregated value is returned. If omitted, the name is
+    equal to the corresponding expression (`expressionX`).
+    If multiple columns have the same name, all but the rightmost column with this
+    name will be ignored.
+
+`expressionX`
+:   An expression that computes an aggregated value.
+
+`grouping_expressionX`
+:   An expression that outputs the values to group by.
+    If its name coincides with one of the existing or computed columns, that column will be overridden by this one.
+
+`boolean_expressionX`
+:   The condition that determines which rows are included when evaluating `expressionX`.
+
+::::{note}
+Individual `null` values are skipped when computing aggregations.
+::::
+
+
+**Description**
+
+The `INLINE STATS` processing command groups rows according to a common value
+(also known as the grouping key), specified after `BY`, and calculates one or more
+aggregated values over the grouped rows. The output table contains the same
+number of rows as the input table. The command only adds new columns or overrides
+existing columns with the same name as the result.
+
+If column names overlap, existing column values may be overridden and column order
+may change. The new columns are added/moved so that they appear in the order
+they are defined in the `INLINE STATS` command.
+
+For the calculation of each aggregated value, the rows in a group can be filtered with
+`WHERE`. If `BY` is omitted the aggregations are applied over the entire dataset.
+
+The following [aggregation functions](/reference/query-languages/esql/functions-operators/aggregation-functions.md) are supported:
+
+:::{include} ../../lists/aggregation-functions.md
+:::
+
+The following [grouping functions](/reference/query-languages/esql/functions-operators/grouping-functions.md) are supported:
+
+* [`BUCKET`](/reference/query-languages/esql/functions-operators/grouping-functions.md#esql-bucket)
+* [`TBUCKET`](/reference/query-languages/esql/functions-operators/grouping-functions.md#esql-tbucket)
+
+
+**Examples**
+
+The following example shows how to calculate a statistic on one column and group
+by the values of another column.
+
+:::{note}
+The `languages` column moves to the last position in the output table because it is
+a column overriden by the `INLINE STATS` command (it's the grouping key) and it is the last column defined by it.
+:::
+
+:::{include} ../examples/inlinestats.csv-spec/max-salary.md
+:::
+
+The following example shows how to calculate an aggregation over the entire dataset
+by omitting `BY`. The order of the existing columns is preserved and a new column
+with the calculated maximum salary value is added as the last column:
+
+:::{include} ../examples/inlinestats.csv-spec/max-salary-without-by.md
+:::
+
+The following example shows how to calculate multiple aggregations with multiple grouping keys:
+
+:::{include} ../examples/inlinestats.csv-spec/multi-agg-multi-grouping.md
+:::
+
+The following example shows how to filter which rows are used for each aggregation, using the `WHERE` clause:
+
+:::{include} ../examples/inlinestats.csv-spec/avg-salaries-where.md
+:::
+
+
+**Limitations**
+
+- The [`CATEGORIZE`](/reference/query-languages/esql/functions-operators/grouping-functions.md#esql-categorize) grouping function is not currently supported.
+- `INLINE STATS` cannot yet have an unbounded [`SORT`](/reference/query-languages/esql/commands/sort.md) before it.
+You must either move the SORT after it, or add a [`LIMIT`](/reference/query-languages/esql/commands/limit.md) before the [`SORT`](/reference/query-languages/esql/commands/sort.md).

docs/reference/query-languages/esql/_snippets/lists/processing-commands.md

@@ -9,6 +9,7 @@
 * [`KEEP`](/reference/query-languages/esql/commands/keep.md)
 * [`LIMIT`](/reference/query-languages/esql/commands/limit.md)
 * [`LOOKUP JOIN`](/reference/query-languages/esql/commands/lookup-join.md)
+* [preview] [`INLINE STATS`](/reference/query-languages/esql/commands/inlinestats-by.md)
 * [preview] [`MV_EXPAND`](/reference/query-languages/esql/commands/mv_expand.md)
 * [`RENAME`](/reference/query-languages/esql/commands/rename.md)
 * [preview] [`RERANK`](/reference/query-languages/esql/commands/rerank.md)

docs/reference/query-languages/esql/commands/inlinestats-by.md

@@ -0,0 +1,10 @@
+---
+navigation_title: "INLINE STATS"
+mapped_pages:
+  - https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-commands.html#esql-inlinestats-by
+---
+
+# `INLINE STATS` [esql-inlinestats-by]
+
+:::{include} ../_snippets/commands/layout/inlinestats-by.md
+:::

docs/reference/query-languages/esql/commands/inlinestats.disabled

@@ -1,102 +0,0 @@
-[discrete]
-[[esql-inlinestats-by]]
-=== `INLINESTATS ... BY`
-
-experimental::["INLINESTATS is highly experimental and only available in SNAPSHOT versions."]
-
-The `INLINESTATS` command calculates an aggregate result and adds new columns
-with the result to the stream of input data.
-
-**Syntax**
-
-[source,esql]
-----
-INLINESTATS [column1 =] expression1[, ..., [columnN =] expressionN]
-[BY grouping_expression1[, ..., grouping_expressionN]]
-----
-
-*Parameters*
-
-`columnX`::
-The name by which the aggregated value is returned. If omitted, the name is
-equal to the corresponding expression (`expressionX`). If multiple columns
-have the same name, all but the rightmost column with this name will be ignored.
-
-`expressionX`::
-An expression that computes an aggregated value. If its name coincides with one
-of the computed columns, that column will be ignored.
-
-`grouping_expressionX`::
-An expression that outputs the values to group by.
-
-NOTE: Individual `null` values are skipped when computing aggregations.
-
-*Description*
-
-The `INLINESTATS` command calculates an aggregate result and merges that result
-back into the stream of input data. Without the optional `BY` clause this will
-produce a single result which is appended to each row. With a `BY` clause this
-will produce one result per grouping and merge the result into the stream based on
-matching group keys.
-
-All of the <<esql-agg-functions,aggregation functions>> are supported.
-
-*Examples*
-
-Find the employees that speak the most languages (it's a tie!):
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/inlinestats.csv-spec[tag=max-languages]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/inlinestats.csv-spec[tag=max-languages-result]
-|===
-
-Find the longest tenured employee who's last name starts with each letter of the alphabet:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/inlinestats.csv-spec[tag=longest-tenured-by-first]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/inlinestats.csv-spec[tag=longest-tenured-by-first-result]
-|===
-
-Find the northern and southern most airports:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/inlinestats.csv-spec[tag=extreme-airports]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/inlinestats.csv-spec[tag=extreme-airports-result]
-|===
-
-NOTE: Our test data doesn't have many "small" airports.
-
-If a `BY` field is multivalued then `INLINESTATS` will put the row in *each*
-bucket like <<esql-stats-by>>:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/inlinestats.csv-spec[tag=mv-group]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/inlinestats.csv-spec[tag=mv-group-result]
-|===
-
-To treat each group key as its own row use <<esql-mv_expand>> before `INLINESTATS`:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/inlinestats.csv-spec[tag=mv-expand]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/inlinestats.csv-spec[tag=mv-expand-result]
-|===

docs/reference/query-languages/esql/functions-operators/aggregation-functions.md

@@ -7,7 +7,7 @@ mapped_pages:
 # {{esql}} aggregation functions [esql-aggregation-functions]
 
 
-The [`STATS`](/reference/query-languages/esql/commands/stats-by.md) command supports these aggregate functions:
+The [`STATS`](/reference/query-languages/esql/commands/stats-by.md) and [`INLINE STATS`](/reference/query-languages/esql/commands/inlinestats-by.md) commands support these aggregate functions:
 
 :::{include} ../_snippets/lists/aggregation-functions.md
 :::

docs/reference/query-languages/esql/functions-operators/grouping-functions.md

@@ -12,6 +12,11 @@ The [`STATS`](/reference/query-languages/esql/commands/stats-by.md) command supp
 :::{include} ../_snippets/lists/grouping-functions.md
 :::
 
+The [`INLINE STATS`](/reference/query-languages/esql/commands/inlinestats-by.md) command supports these grouping functions:
+
+* [`BUCKET`](/reference/query-languages/esql/functions-operators/grouping-functions.md#esql-bucket)
+* [`TBUCKET`](/reference/query-languages/esql/functions-operators/grouping-functions.md#esql-tbucket)
+
 
 :::{include} ../_snippets/functions/layout/bucket.md
 :::
@@ -19,6 +24,7 @@ The [`STATS`](/reference/query-languages/esql/commands/stats-by.md) command supp
 :::{include} ../_snippets/functions/layout/tbucket.md
 :::
 
+
 :::{note}
 The `CATEGORIZE` function requires a [platinum license](https://www.elastic.co/subscriptions).
 :::

docs/reference/query-languages/esql/limitations.md

@@ -242,6 +242,13 @@ Work around this limitation by converting the field to single value with one of
 {{esql}} only supports the UTC timezone.
 
 
+## INLINE STATS limitations [esql-limitations-inlinestats]
+
+[`CATEGORIZE`](/reference/query-languages/esql/functions-operators/grouping-functions.md#esql-categorize) grouping function is not currently supported.
+
+Also, [`INLINE STATS`](/reference/query-languages/esql/commands/inlinestats-by.md) cannot yet have an unbounded [`SORT`](/reference/query-languages/esql/commands/sort.md) before it. You must either move the SORT after it, or add a [`LIMIT`](/reference/query-languages/esql/commands/limit.md) before the [`SORT`](/reference/query-languages/esql/commands/sort.md).
+
+
 ## Kibana limitations [esql-limitations-kibana]
 
 * The user interface to filter data is not enabled when Discover is in {{esql}} mode. To filter data, write a query that uses the [`WHERE`](/reference/query-languages/esql/commands/where.md) command instead.

docs/reference/query-languages/toc.yml

@@ -107,6 +107,7 @@ toc:
                   - file: esql/commands/eval.md
                   - file: esql/commands/fork.md
                   - file: esql/commands/grok.md
+                  - file: esql/commands/inlinestats-by.md
                   - file: esql/commands/keep.md
                   - file: esql/commands/limit.md
                   - file: esql/commands/lookup-join.md

x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec

@@ -13,24 +13,110 @@ eth0           |epsilon gw instance|epsilon        |[fe80::cae2:65ff:fece:feb9,
 
 maxOfInt
 required_capability: inlinestats_v11
-// tag::max-languages[]
+
 FROM employees
 | KEEP emp_no, languages
 | INLINESTATS max_lang = MAX(languages) 
 | WHERE max_lang == languages
-// end::max-languages[]
 | SORT emp_no ASC
 | LIMIT 5
 ;
 
-// tag::max-languages-result[]
 emp_no:integer | languages:integer | max_lang:integer
          10002 |                 5 | 5
          10004 |                 5 | 5
          10011 |                 5 | 5
          10012 |                 5 | 5
          10014 |                 5 | 5
-// end::max-languages-result[]
+;
+
+docsMaxWithoutBy
+required_capability: inlinestats_v11
+// tag::max-salary-without-by[]
+FROM employees
+| KEEP emp_no, languages, salary
+| INLINESTATS max_salary = MAX(salary) 
+// end::max-salary-without-by[]
+| SORT emp_no ASC
+| LIMIT 5
+;
+
+// tag::max-salary-without-by-result[]
+emp_no:integer | languages:integer | salary:integer | max_salary:integer
+10001          |2                  |57305           |74999          
+10002          |5                  |56371           |74999          
+10003          |4                  |61805           |74999          
+10004          |5                  |36174           |74999          
+10005          |1                  |63528           |74999
+// end::max-salary-without-by-result[]
+;
+
+docsMax
+required_capability: inlinestats_v11
+// tag::max-salary[]
+FROM employees
+| KEEP emp_no, languages, salary
+| INLINESTATS max_salary = MAX(salary) BY languages
+// end::max-salary[]
+| SORT emp_no ASC
+| LIMIT 5
+;
+
+// tag::max-salary-result[]
+emp_no:integer | salary:integer | max_salary:integer | languages:integer
+10001          |57305           |73578               |2              
+10002          |56371           |66817               |5              
+10003          |61805           |74572               |4              
+10004          |36174           |66817               |5              
+10005          |63528           |73717               |1
+// end::max-salary-result[]
+;
+
+docsMultiAggsMultiGroupings
+required_capability: inlinestats_v11
+// tag::multi-agg-multi-grouping[]
+FROM employees
+| WHERE still_hired
+| KEEP emp_no, languages, salary, hire_date
+| EVAL tenure = DATE_DIFF("year", hire_date, now())
+| DROP hire_date
+| INLINESTATS avg_salary = AVG(salary), count = count(*) BY languages, tenure
+// end::multi-agg-multi-grouping[]
+| SORT emp_no
+| LIMIT 5
+;
+
+// tag::multi-agg-multi-grouping-result[]
+ emp_no:integer | salary:integer | avg_salary:double | count:long | languages:integer | tenure:integer
+10001           |57305           |51130.5            |2           |2                  |39
+10002           |56371           |40180.0            |3           |5                  |39
+10004           |36174           |30749.0            |2           |5                  |38
+10005           |63528           |63528.0            |1           |1                  |36
+10007           |74572           |58644.0            |2           |4                  |36
+// end::multi-agg-multi-grouping-result[]
+;
+
+docsInlinestatsWithWhere
+required_capability: inlinestats_v11
+// tag::avg-salaries-where[]
+FROM employees
+| KEEP emp_no, salary
+| INLINESTATS avg_lt_50 = ROUND(AVG(salary)) WHERE salary < 50000,
+              avg_lt_60 = ROUND(AVG(salary)) WHERE salary >=50000 AND salary < 60000,
+              avg_gt_60 = ROUND(AVG(salary)) WHERE salary >= 60000
+// end::avg-salaries-where[]
+| SORT emp_no
+| LIMIT 5
+;
+
+// tag::avg-salaries-where-result[]
+ emp_no:integer | salary:integer | avg_lt_50:double | avg_lt_60:double | avg_gt_60:double
+10001           |57305           |38292.0           |54221.0           |67286.0
+10002           |56371           |38292.0           |54221.0           |67286.0
+10003           |61805           |38292.0           |54221.0           |67286.0
+10004           |36174           |38292.0           |54221.0           |67286.0
+10005           |63528           |38292.0           |54221.0           |67286.0
+// end::avg-salaries-where-result[]
 ;
 
 maxOfIntByKeyword
@@ -41,7 +127,8 @@ FROM employees
 | INLINESTATS max_lang = MAX(languages) BY gender 
 | WHERE max_lang == languages
 | SORT emp_no ASC
-| LIMIT 5;
+| LIMIT 5
+;
 
 emp_no:integer | languages:integer | max_lang:integer | gender:keyword
          10002 |                 5 | 5                | F
@@ -111,24 +198,20 @@ emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_secon
 maxOfLongByCalculatedKeyword
 required_capability: inlinestats_v11
 
-// tag::longest-tenured-by-first[]
 FROM employees
 | KEEP emp_no, avg_worked_seconds, last_name
 | INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY SUBSTRING(last_name, 0, 1)
 | WHERE max_avg_worked_seconds == avg_worked_seconds
-// end::longest-tenured-by-first[]
 | SORT last_name ASC
 | LIMIT 5
 ;
 
-// tag::longest-tenured-by-first-result[]
 emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_seconds:long | SUBSTRING(last_name, 0, 1):keyword
          10065 |               372660279 | Awdeh             | 372660279                   | A
          10074 |               382397583 | Bernatsky         | 382397583                   | B
          10044 |               387408356 | Casley            | 387408356                   | C
          10030 |               394597613 | Demeyer           | 394597613                   | D
          10087 |               305782871 | Eugenio           | 305782871                   | E
-// end::longest-tenured-by-first-result[]
 ;
 
 maxOfLongByCalculatedNamedKeyword
@@ -298,45 +381,36 @@ from employees
 byMultivaluedSimple
 required_capability: inlinestats_v11
 
-// tag::mv-group[]
 FROM airports
 | INLINESTATS min_scalerank=MIN(scalerank) BY type
 | EVAL type=MV_SORT(type), min_scalerank=MV_SORT(min_scalerank)
 | KEEP abbrev, type, scalerank, min_scalerank
 | WHERE abbrev == "GWL"
-// end::mv-group[]
 ;
 
-// tag::mv-group-result[]
 abbrev:keyword |  type:keyword   | scalerank:integer | min_scalerank:integer
            GWL | [mid, military] | 9                 | [2, 4]
-// end::mv-group-result[]
 ;
 
 byMultivaluedMvExpand
 required_capability: inlinestats_v11
 
-// tag::mv-expand[]
 FROM airports
 | KEEP abbrev, type, scalerank
 | MV_EXPAND type
 | INLINESTATS min_scalerank=MIN(scalerank) BY type
 | SORT min_scalerank ASC
 | WHERE abbrev == "GWL"
-// end::mv-expand[]
 ;
 
-// tag::mv-expand-result[]
 abbrev:keyword | scalerank:integer | min_scalerank:integer | type:keyword 
 GWL            |9                  |2                      |mid            
 GWL            |9                  |4                      |military
-// end::mv-expand-result[]
 ;
 
 byMvExpand
 required_capability: inlinestats_v11
 
-// tag::extreme-airports[]
 FROM airports
 | MV_EXPAND type
 | EVAL lat = ST_Y(location)
@@ -344,10 +418,8 @@ FROM airports
 | WHERE lat == most_northern OR lat == most_southern
 | SORT lat DESC
 | KEEP type, name, location
-// end::extreme-airports[]
 ;
 
-// tag::extreme-airports-result[]
  type:keyword |           name:text           | location:geo_point
           mid |             Svalbard Longyear | POINT (15.495229 78.246717)
         major |                Tromsø Langnes | POINT (18.9072624292132 69.6796790473478)
@@ -359,7 +431,6 @@ FROM airports
      military |         Santos Air Force Base | POINT (-46.3052704931003 -23.9237590410637)
         major |            Christchurch Int'l | POINT (172.538675565223 -43.4885486784104)
           mid |          Hermes Quijada Int'l | POINT (-67.7530268462675 -53.7814746058316)
-// end::extreme-airports-result[]
 ;
 
 mvMinMvExpand
@@ -1915,6 +1986,7 @@ inlinestatsWithFalseFiltersFromRow
 //  null      |2
 //  null      |3
 //  null      |4
+// TODO: add docs about whatever behavior is decided here
 required_capability: inlinestats_v11
 row x = null, a = 1, b = [2,3,4]
 | inlinestats c=max(a) where x