ESQ: Add metadata fields documentation (#98991)

This adds documentation for the supported metadata fields.

---------

Co-authored-by: Abdon Pijpelink <abdon.pijpelink@elastic.co>

docs/reference/esql/index.asciidoc

@@ -135,6 +135,8 @@ include::aggregation-functions.asciidoc[]
 
 include::multivalued-fields.asciidoc[]
 
+include::metadata-fields.asciidoc[]
+
 include::task-management.asciidoc[]
 
 :esql-tests!:

docs/reference/esql/metadata-fields.asciidoc

@@ -0,0 +1,55 @@
+[[esql-metadata-fields]]
+== {esql} metadata fields
+
+++++
+<titleabbrev>Metadata fields</titleabbrev>
+++++
+
+{esql} can access <<mapping-fields, metadata fields>>. The currently
+supported ones are:
+
+  * <<mapping-index-field,`_index`>>: the index to which the document belongs.
+  The field is of the type <<keyword, keyword>>.
+
+  * <<mapping-id-field,`_id`>>: the source document's ID. The field is of the
+  type <<keyword, keyword>>.
+
+  * `_version`: the source document's version. The field is of the type
+  <<number,long>>.
+
+To enable the access to these fields, the <<esql-from,`FROM`>> source command needs
+to be provided with a dedicated directive:
+
+[source,esql]
+----
+FROM index [METADATA _index, _id]
+----
+
+Metadata fields are only available if the source of the data is an index.
+Consequently, `FROM` is the only source commands that supports the `METADATA`
+directive.
+
+Once enabled, the fields are then available to subsequent processing commands, just
+like the other index fields:
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/metadata-ignoreCsvTests.csv-spec[tag=multipleIndices]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/metadata-ignoreCsvTests.csv-spec[tag=multipleIndices-result]
+|===
+
+Also, similar to the index fields, once an aggregation is performed, a
+metadata field will no longer be accessible to subsequent commands, unless
+used as grouping field:
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/metadata-ignoreCsvTests.csv-spec[tag=metaIndexInAggs]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/metadata-ignoreCsvTests.csv-spec[tag=metaIndexInAggs-result]
+|===

docs/reference/esql/source-commands/from.asciidoc

@@ -27,3 +27,10 @@ or aliases:
 ----
 FROM employees-00001,employees-*
 ----
+
+Use the `METADATA` directive to enable <<esql-metadata-fields,metadata fields>>:
+
+[source,esql]
+----
+FROM employees [METADATA _id]
+----

x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata-ignoreCsvTests.csv-spec

@@ -24,10 +24,16 @@ emp_no:integer
 ;
 
 metaIndexInAggs
-from employees [metadata _index] | stats max = max(emp_no) by _index;
+// tag::metaIndexInAggs[]
+FROM employees [METADATA _index, _id]
+| STATS max = MAX(emp_no) BY _index
+// end::metaIndexInAggs[]
+;
 
+// tag::metaIndexInAggs-result[]
 max:integer |_index:keyword
 10100       |employees
+// end::metaIndexInAggs-result[]
 ;
 
 metaIndexAliasedInAggs
@@ -103,3 +109,23 @@ emp_no:integer |_index:integer |_version:keyword
 10002          |3              |version
 10003          |3              |version
 ;
+
+multipleIndices
+// tag::multipleIndices[]
+FROM ul_logs, apps [METADATA _index, _version]
+| WHERE id IN (13, 14) AND _version == 1
+| EVAL key = CONCAT(_index, "_", TO_STR(id))
+| SORT id, _index
+| KEEP id, _index, _version, key
+// end::multipleIndices[]
+;
+
+// tag::multipleIndices-result[]
+ id:long | _index:keyword | _version:long | key:keyword
+13       |apps            |1              |apps_13        
+13       |ul_logs         |1              |ul_logs_13     
+14       |apps            |1              |apps_14        
+14       |ul_logs         |1              |ul_logs_14     
+    
+// end::multipleIndices-result[]
+;