Add improvements to the ES|QL docs (#101195)

Content and structural improvements to the ES|QL docs

---------

Co-authored-by: Alexandros Batsakis <abatsakis@splunk.com>
Co-authored-by: Abdon Pijpelink <abdon.pijpelink@elastic.co>

docs/reference/esql/esql-commands.asciidoc

@@ -1,17 +1,63 @@
 [[esql-commands]]
-== {esql} commands
+=== {esql} commands
 
 ++++
 <titleabbrev>Commands</titleabbrev>
 ++++
 
-{esql} provides a comprehensive set of source and processing commands:
+// tag::source_commands[]
+==== Source commands
 
-<<esql-source-commands>>::
-include::source-commands/esql-source-commands.asciidoc[tag=list]
+An {esql} source command produces a table, typically with data from {es}.
 
-<<esql-processing-commands>>::
-include::processing-commands/esql-processing-commands.asciidoc[tag=list]
+image::images/esql/source-command.svg[A source command producing a table from {es},align="center"]
 
-include::source-commands/esql-source-commands.asciidoc[]
-include::processing-commands/esql-processing-commands.asciidoc[]
+{esql} supports these source commands:
+
+* <<esql-from>>
+* <<esql-row>>
+* <<esql-show>>
+
+include::source-commands/from.asciidoc[]
+include::source-commands/row.asciidoc[]
+include::source-commands/show.asciidoc[]
+
+// end::source_command[]
+
+// tag::proc_commands[]
+==== Processing commands
+
+{esql} processing commands change an input table by adding, removing, or changing
+rows and columns.
+
+image::images/esql/processing-command.svg[A processing command changing an input table,align="center"]
+
+{esql} supports these processing commands:
+
+* <<esql-dissect>>
+* <<esql-drop>>
+* <<esql-enrich>>
+* <<esql-eval>>
+* <<esql-grok>>
+* <<esql-keep>>
+* <<esql-limit>>
+* <<esql-mv_expand>>
+* <<esql-rename>>
+* <<esql-sort>>
+* <<esql-stats-by>>
+* <<esql-where>>
+
+include::processing-commands/dissect.asciidoc[]
+include::processing-commands/drop.asciidoc[]
+include::processing-commands/enrich.asciidoc[]
+include::processing-commands/eval.asciidoc[]
+include::processing-commands/grok.asciidoc[]
+include::processing-commands/keep.asciidoc[]
+include::processing-commands/limit.asciidoc[]
+include::processing-commands/mv_expand.asciidoc[]
+include::processing-commands/rename.asciidoc[]
+include::processing-commands/sort.asciidoc[]
+include::processing-commands/stats.asciidoc[]
+include::processing-commands/where.asciidoc[]
+
+// end::proc_command[]

docs/reference/esql/esql-enrich-data.asciidoc

@@ -82,22 +82,22 @@ your query.
 ====
 
 [[esql-enrich-prereqs]]
-===== Prerequisites
+==== Prerequisites
 
 include::{es-repo-dir}/ingest/apis/enrich/put-enrich-policy.asciidoc[tag=enrich-policy-api-prereqs]
 
 [[esql-create-enrich-source-index]]
-===== Add enrich data
+==== Add enrich data
 
 include::../ingest/enrich.asciidoc[tag=create-enrich-source-index]
 
 [[esql-create-enrich-policy]]
-===== Create an enrich policy
+==== Create an enrich policy
 
 include::../ingest/enrich.asciidoc[tag=create-enrich-policy]
 
 [[esql-execute-enrich-policy]]
-===== Execute the enrich policy
+==== Execute the enrich policy
 
 include::../ingest/enrich.asciidoc[tag=execute-enrich-policy1]
 

docs/reference/esql/esql-functions-operators.asciidoc

@@ -1,36 +1,37 @@
 [[esql-functions-operators]]
-== {esql} functions and operators
+=== {esql} functions and operators
 
 ++++
 <titleabbrev>Functions and operators</titleabbrev>
 ++++
 
-{esql} provides a comprehensive set of functions and operators:
+{esql} provides a comprehensive set of functions and operators for working with data.
+The functions are divided into the following categories:
 
 [[esql-functions]]
 <<esql-agg-functions>>::
-include::functions/aggregation-functions.asciidoc[tag=list]
+include::functions/aggregation-functions.asciidoc[tag=agg_list]
 
 <<esql-math-functions>>::
-include::functions/math-functions.asciidoc[tag=list]
+include::functions/math-functions.asciidoc[tag=math_list]
 
 <<esql-string-functions>>::
-include::functions/string-functions.asciidoc[tag=list]
+include::functions/string-functions.asciidoc[tag=string_list]
 
 <<esql-date-time-functions>>::
-include::functions/date-time-functions.asciidoc[tag=list]
+include::functions/date-time-functions.asciidoc[tag=date_list]
 
 <<esql-type-conversion-functions>>::
-include::functions/type-conversion-functions.asciidoc[tag=list]
+include::functions/type-conversion-functions.asciidoc[tag=type_list]
 
 <<esql-conditional-functions-and-expressions>>::
-include::functions/conditional-functions-and-expressions.asciidoc[tag=list]
+include::functions/conditional-functions-and-expressions.asciidoc[tag=cond_list]
 
 <<esql-mv-functions>>::
-include::functions/mv-functions.asciidoc[tag=list]
+include::functions/mv-functions.asciidoc[tag=mv_list]
 
 <<esql-operators>>::
-include::functions/operators.asciidoc[tag=list]
+include::functions/operators.asciidoc[tag=op_list]
 
 include::functions/aggregation-functions.asciidoc[]
 include::functions/math-functions.asciidoc[]

docs/reference/esql/esql-kibana.asciidoc

@@ -5,4 +5,11 @@
 <titleabbrev>Kibana</titleabbrev>
 ++++
 
-coming::[8.11]
+
+Use {esql} in Discover to explore a data set. From the data view dropdown,
+select *Try {esql}* to get started.
+
+NOTE: {esql} queries in Discover and Lens are subject to the time range selected
+with the time filter.
+
+

docs/reference/esql/esql-language.asciidoc

@@ -1,18 +1,23 @@
 [[esql-language]]
-== {esql} language
+== Working with the {esql} language
 
 ++++
-<titleabbrev>{esql} language</titleabbrev>
+<titleabbrev>Working with the {esql} language</titleabbrev>
 ++++
 
 Detailed information about the {esql} language:
 
 * <<esql-syntax>>
+* <<esql-commands>>
+* <<esql-functions>>
 * <<esql-multivalued-fields>>
 * <<esql-metadata-fields>>
 * <<esql-enrich-data>>
 
 include::esql-syntax.asciidoc[]
+include::esql-commands.asciidoc[]
+include::esql-functions-operators.asciidoc[]
 include::multivalued-fields.asciidoc[]
 include::metadata-fields.asciidoc[]
-include::esql-enrich-data.asciidoc[]
+include::esql-enrich-data.asciidoc[]
+

docs/reference/esql/esql-query-api.asciidoc

@@ -6,7 +6,20 @@
 
 Returns search results for an <<esql,ES|QL ({es} query language)>> query.
 
-include::index.asciidoc[tag=esql-query-api-example]
+[source,console]
+----
+POST /_query
+{
+  "query": """
+    FROM library
+    | EVAL year = DATE_TRUNC(1 YEARS, release_date)
+    | STATS MAX(page_count) BY year
+    | SORT year
+    | LIMIT 5
+  """
+}
+----
+// TEST[setup:library]
 
 [discrete]
 [[esql-query-api-request]]

docs/reference/esql/esql-rest.asciidoc

@@ -45,6 +45,7 @@ highly recommended), take advantage of the triple quotes `"""` when creating the
 query. This not only automatically escapes double quotes (`"`) inside the query
 string but also supports multi-line requests:
 
+// tag::esql-query-api[]
 [source,console]
 ----
 POST /_query?format=txt

docs/reference/esql/esql-syntax.asciidoc

@@ -9,8 +9,8 @@
 [[esql-basic-syntax]]
 === Basic syntax
 
-An {esql} query is composed of a <<esql-source-commands,source command>> followed
-by an optional series of <<esql-processing-commands,processing commands>>,
+An {esql} query is composed of a <<esql-commands,source_command>> followed
+by an optional series of <<esql-commands,processing commands>>,
 separated by a pipe character: `|`. For example:
 
 [source,esql]

docs/reference/esql/functions/aggregation-functions.asciidoc

@@ -1,5 +1,5 @@
 [[esql-agg-functions]]
-=== {esql} aggregate functions
+==== {esql} aggregate functions
 
 ++++
 <titleabbrev>Aggregate functions</titleabbrev>
@@ -7,7 +7,7 @@
 
 The <<esql-stats-by>> function supports these aggregate functions:
 
-// tag::list[]
+// tag::agg_list[]
 * <<esql-agg-avg>>
 * <<esql-agg-count>>
 * <<esql-agg-count-distinct>>
@@ -17,7 +17,7 @@ The <<esql-stats-by>> function supports these aggregate functions:
 * <<esql-agg-min>>
 * <<esql-agg-percentile>>
 * <<esql-agg-sum>>
-// end::list[]
+// end::agg_list[]
 
 include::avg.asciidoc[]
 include::count.asciidoc[]

docs/reference/esql/functions/conditional-functions-and-expressions.asciidoc

@@ -1,19 +1,19 @@
 [[esql-conditional-functions-and-expressions]]
-=== {esql} conditional functions and expressions
+==== {esql} conditional functions and expressions
 
 ++++
 <titleabbrev>Conditional functions and expressions</titleabbrev>
 ++++
 
 Conditional functions return one of their arguments by evaluating in an if-else
-manner. {esql} supports these conditional functions: 
+manner. {esql} supports these conditional functions:
 
-// tag::list[]
+// tag::cond_list[]
 * <<esql-case>>
 * <<esql-coalesce>>
 * <<esql-greatest>>
 * <<esql-least>>
-// end::list[]
+// end::cond_list[]
 
 include::case.asciidoc[]
 include::coalesce.asciidoc[]

docs/reference/esql/functions/date-time-functions.asciidoc

@@ -1,5 +1,5 @@
 [[esql-date-time-functions]]
-=== {esql} date-time functions
+==== {esql} date-time functions
 
 ++++
 <titleabbrev>Date-time functions</titleabbrev>
@@ -7,14 +7,14 @@
 
 {esql} supports these date-time functions:
 
-// tag::list[]
+// tag::date_list[]
 * <<esql-auto_bucket>>
 * <<esql-date_extract>>
 * <<esql-date_format>>
 * <<esql-date_parse>>
 * <<esql-date_trunc>>
 * <<esql-now>>
-// end::list[]
+// end::date_list[]
 
 include::auto_bucket.asciidoc[]
 include::date_extract.asciidoc[]

docs/reference/esql/functions/math-functions.asciidoc

@@ -1,5 +1,5 @@
 [[esql-math-functions]]
-=== {esql} mathematical functions
+==== {esql} mathematical functions
 
 ++++
 <titleabbrev>Mathematical functions</titleabbrev>
@@ -7,7 +7,7 @@
 
 {esql} supports these mathematical functions:
 
-// tag::list[]
+// tag::math_list[]
 * <<esql-abs>>
 * <<esql-acos>>
 * <<esql-asin>>
@@ -28,7 +28,7 @@
 * <<esql-tan>>
 * <<esql-tanh>>
 * <<esql-tau>>
-// end::list[]
+// end::math_list[]
 
 include::abs.asciidoc[]
 include::acos.asciidoc[]

docs/reference/esql/functions/mv-functions.asciidoc

@@ -1,5 +1,5 @@
 [[esql-mv-functions]]
-=== {esql} multivalue functions
+==== {esql} multivalue functions
 
 ++++
 <titleabbrev>Multivalue functions</titleabbrev>
@@ -7,7 +7,7 @@
 
 {esql} supports these multivalue functions:
 
-// tag::list[]
+// tag::mv_list[]
 * <<esql-mv_avg>>
 * <<esql-mv_concat>>
 * <<esql-mv_count>>
@@ -16,7 +16,7 @@
 * <<esql-mv_median>>
 * <<esql-mv_min>>
 * <<esql-mv_sum>>
-// end::list[]
+// end::mv_list[]
 
 include::mv_avg.asciidoc[]
 include::mv_concat.asciidoc[]

docs/reference/esql/functions/operators.asciidoc

@@ -1,5 +1,5 @@
 [[esql-operators]]
-=== {esql} operators
+==== {esql} operators
 
 ++++
 <titleabbrev>Operators</titleabbrev>
@@ -7,7 +7,7 @@
 
 Boolean operators for comparing against one or multiple expressions.
 
-// tag::list[]
+// tag::op_list[]
 * <<esql-binary-operators>>
 * <<esql-logical-operators>>
 * <<esql-predicates>>
@@ -20,7 +20,7 @@ Boolean operators for comparing against one or multiple expressions.
 * <<esql-like-operator>>
 * <<esql-rlike-operator>>
 * <<esql-starts_with>>
-// end::list[]
+// end::op_list[]
 
 include::binary.asciidoc[]
 include::logical.asciidoc[]

docs/reference/esql/functions/string-functions.asciidoc

@@ -1,5 +1,5 @@
 [[esql-string-functions]]
-=== {esql} string functions
+==== {esql} string functions
 
 ++++
 <titleabbrev>String functions</titleabbrev>
@@ -7,7 +7,7 @@
 
 {esql} supports these string functions:
 
-// tag::list[]
+// tag::string_list[]
 * <<esql-concat>>
 * <<esql-left>>
 * <<esql-length>>
@@ -18,7 +18,7 @@
 * <<esql-split>>
 * <<esql-substring>>
 * <<esql-trim>>
-// end::list[]
+// end::string_list[]
 
 include::concat.asciidoc[]
 include::left.asciidoc[]

docs/reference/esql/functions/type-conversion-functions.asciidoc

@@ -1,5 +1,5 @@
 [[esql-type-conversion-functions]]
-=== {esql} type conversion functions
+==== {esql} type conversion functions
 
 ++++
 <titleabbrev>Type conversion functions</titleabbrev>
@@ -7,7 +7,7 @@
 
 {esql} supports these type conversion functions:
 
-// tag::list[]
+// tag::type_list[]
 * <<esql-to_boolean>>
 * <<esql-to_datetime>>
 * <<esql-to_degrees>>
@@ -19,7 +19,7 @@
 * <<esql-to_string>>
 * <<esql-to_unsigned_long>>
 * <<esql-to_version>>
-// end::list[]
+// end::type_list[]
 
 include::to_boolean.asciidoc[]
 include::to_datetime.asciidoc[]

docs/reference/esql/index.asciidoc

@@ -5,22 +5,36 @@
 :esql-specs: {esql-tests}/testFixtures/src/main/resources
 
 [partintro]
---
 
 preview::[]
 
-The {es} Query Language ({esql}) is a query language that enables the iterative
-exploration of data.
+The {es} Query Language ({esql}) provides a powerful way to filter, transform, and analyze data stored in {es}.
+Users can author {esql} queries to find specific events, perform statistical analysis, and generate visualizations.
+It supports a wide range of commands and functions that enable users to perform various data operations,
+such as filtering, aggregation, time-series analysis, and more.
 
-An {esql} query consists of a series of commands, separated by pipes. Each query
-starts with a <<esql-source-commands,source command>>. A source command produces
+The {es} Query Language ({esql}) makes use of "pipes" to manipulate and transform data in a step-by-step fashion.
+This approach allows users to compose a series of operations, where the output of one operation becomes the input for the next,
+enabling complex data transformations and analysis.
+
+A simple example of an {esql} query is shown below:
+[source,esql]
+----
+FROM employees
+| EVAL age = DATE_DIFF(NOW(), birth_date, 'Y')
+| STATS AVG(age) BY department
+| SORT age DESC
+----
+
+Each {esql} query starts with a <<esql-commands,source command>>. A source command produces
 a table, typically with data from {es}.
 
 image::images/esql/source-command.svg[A source command producing a table from {es},align="center"]
 
 A source command can be followed by one or more
-<<esql-processing-commands,processing commands>>. Processing commands change an
+<<esql-commands,processing commands>>. Processing commands change an
 input table by adding, removing, or changing rows and columns.
+Processing commands can perform filtering, projection, aggregation, and more.
 
 image::images/esql/processing-command.svg[A processing command changing an input table,align="center"]
 
@@ -32,78 +46,19 @@ image::images/esql/chaining-processing-commands.svg[Processing commands can be c
 The result of a query is the table produced by the final processing command.
 
 [discrete]
-[[esql-console]]
-=== Run an {esql} query
+=== The {esql} Compute Engine
 
-[discrete]
-==== The {esql} API
+{esql} is more than a language. It represents a significant investment in new compute capabilities within {es}.
+To achieve both the functional and performance requirements for {esql}, it was necessary to build an entirely new
+compute architecture. {esql} search, aggregation, and transformation functions are directly executed within Elasticsearch
+itself. Query expressions are not transpiled to Query DSL for execution. This approach allows {esql} to be extremely performant and versatile.
 
-Use the <<esql-rest,`_query` endpoint>> to run an {esql} query:
-
-// tag::esql-query-api-example[]
-[source,console]
-----
-POST /_query
-{
-  "query": """
-    FROM library
-    | EVAL year = DATE_TRUNC(1 YEARS, release_date)
-    | STATS MAX(page_count) BY year
-    | SORT year
-    | LIMIT 5
-  """
-}
-----
-// TEST[setup:library]
-// end::esql-query-api-example[]
-
-The results come back in rows:
-
-[source,console-result]
-----
-{
-  "columns": [
-    { "name": "MAX(page_count)", "type": "integer"},
-    { "name": "year"           , "type": "date"}
-  ],
-  "values": [
-    [268, "1932-01-01T00:00:00.000Z"],
-    [224, "1951-01-01T00:00:00.000Z"],
-    [227, "1953-01-01T00:00:00.000Z"],
-    [335, "1959-01-01T00:00:00.000Z"],
-    [604, "1965-01-01T00:00:00.000Z"]
-  ]
-}
-----
-
-By default, results are returned as JSON. You can return data in other
-<<esql-rest-format,response formats>> by specifying the `format` parameter in
-the URL or by setting the `Accept` or `Content-Type` HTTP header.
-
-By default, an {esql} query returns up to 500 rows. You can change this using
-the <<esql-limit,`LIMIT` command>>. The previous query's `LIMIT` command limits
-results to 5 rows. The maximum number of returned rows is 10,000 rows,
-regardless of the `LIMIT` value.
-
-[discrete]
-==== {kib}
-
-Use {esql} in Discover to explore a data set. From the data view dropdown,
-select *Try {esql}* to get started.
-
-NOTE: {esql} queries in Discover and Lens are subject to the time range selected
-with the time filter.
-
---
+The new {esql} execution engine was designed with performance in mind — it operates on blocks at a time instead of per row, targets vectorization and cache locality, and embraces specialization and multi-threading. It is a separate component from the existing Elasticsearch aggregation framework with different performance characteristics.
 
 include::esql-get-started.asciidoc[]
 
 include::esql-language.asciidoc[]
 
-include::esql-commands.asciidoc[]
-
-include::esql-functions-operators.asciidoc[]
-
 include::esql-rest.asciidoc[]
 
 include::esql-kibana.asciidoc[]

docs/reference/esql/processing-commands/esql-processing-commands.asciidoc

@@ -1,41 +0,0 @@
-[[esql-processing-commands]]
-=== {esql} processing commands
-
-++++
-<titleabbrev>Processing commands</titleabbrev>
-++++
-
-{esql} processing commands change an input table by adding, removing, or changing
-rows and columns.
-
-image::images/esql/processing-command.svg[A processing command changing an input table,align="center"]
-
-{esql} supports these processing commands:
-
-// tag::list[]
-* <<esql-dissect>>
-* <<esql-drop>>
-* <<esql-enrich>>
-* <<esql-eval>>
-* <<esql-grok>>
-* <<esql-keep>>
-* <<esql-limit>>
-* <<esql-mv_expand>>
-* <<esql-rename>>
-* <<esql-sort>>
-* <<esql-stats-by>>
-* <<esql-where>>
-// end::list[]
-
-include::dissect.asciidoc[]
-include::drop.asciidoc[]
-include::enrich.asciidoc[]
-include::eval.asciidoc[]
-include::grok.asciidoc[]
-include::keep.asciidoc[]
-include::limit.asciidoc[]
-include::mv_expand.asciidoc[]
-include::rename.asciidoc[]
-include::sort.asciidoc[]
-include::stats.asciidoc[]
-include::where.asciidoc[]

docs/reference/esql/processing-commands/stats.asciidoc

@@ -42,4 +42,4 @@ include::{esql-specs}/docs.csv-spec[tag=statsGroupByMultipleValues]
 
 The following aggregation functions are supported:
 
-include::../functions/aggregation-functions.asciidoc[tag=list]
+include::../functions/aggregation-functions.asciidoc[tag=agg_list]

docs/reference/esql/source-commands/esql-source-commands.asciidoc

@@ -1,22 +0,0 @@
-[[esql-source-commands]]
-=== {esql} source commands
-
-++++
-<titleabbrev>Source commands</titleabbrev>
-++++
-
-An {esql} source command produces a table, typically with data from {es}.
-
-image::images/esql/source-command.svg[A source command producing a table from {es},align="center"]
-
-{esql} supports these source commands:
-
-// tag::list[]
-* <<esql-from>>
-* <<esql-row>>
-* <<esql-show>>
-// end::list[]
-
-include::from.asciidoc[]
-include::row.asciidoc[]
-include::show.asciidoc[]

docs/reference/esql/task-management.asciidoc

@@ -5,7 +5,7 @@
 <titleabbrev>Task management</titleabbrev>
 ++++
 
-You can get running {esql} queries with the <<tasks,task management API>>:
+You can list running {esql} queries with the <<tasks,task management API>>:
 
 [source,console,id=esql-task-management-get-all]
 ----