ESQL: Grammar - FROM METADATA no longer require [] (#105221)

Remove usage of [ ] through-out the grammar, in this case inside
 FROM METADATA.

docs/changelog/105221.yaml

@@ -0,0 +1,14 @@
+pr: 105221
+summary: "ESQL: Grammar - FROM METADATA no longer requires []"
+area: ES|QL
+type: breaking
+issues: []
+breaking:
+  title: "ESQL: Grammar - FROM METADATA no longer requires []"
+  area: REST API
+  details: "Remove [ ] for METADATA option inside FROM command statements"
+  impact: "Previously to return metadata fields, one had to use square brackets:\
+    \ (eg. 'FROM index [METADATA _index]').\
+    \ This is no longer needed: the [ ] are dropped and do not have to be specified,\
+    \ thus simplifying the command above to:'FROM index METADATA _index'."
+  notable: false

docs/reference/esql/metadata-fields.asciidoc

@@ -5,10 +5,10 @@
 <titleabbrev>Metadata fields</titleabbrev>
 ++++
 
-{esql} can access <<mapping-fields, metadata fields>>. The following are currently
-supported:
+{esql} can access <<mapping-fields, metadata fields>>. The currently
+supported ones are:
 
-  * <<mapping-index-field,`_index`>>: the index the document belongs to.
+  * <<mapping-index-field,`_index`>>: the index to which the document belongs.
   The field is of the type <<keyword, keyword>>.
 
   * <<mapping-id-field,`_id`>>: the source document's ID. The field is of the
@@ -17,18 +17,12 @@ supported:
   * `_version`: the source document's version. The field is of the type
   <<number,long>>.
 
-   * <<mapping-source-field,`_source`>>: the original JSON document body 
-  that was passed at index time (or a reconstructed version if 
-  <<synthetic-source, synthetic `_source_`>> is enabled). The field is 
-  loaded as a special `_source` type. This field is not supported by 
-  functions.
-
-To enable access to these fields, the <<esql-from,`FROM`>> source command 
-requires a dedicated directive:
+To enable the access to these fields, the <<esql-from,`FROM`>> source command needs
+to be provided with a dedicated directive:
 
 [source,esql]
 ----
-FROM index [METADATA _index, _id, _source]
+FROM index METADATA _index, _id
 ----
 
 Metadata fields are only available if the source of the data is an index.

docs/reference/esql/source-commands/from.asciidoc

@@ -66,9 +66,9 @@ or aliases:
 FROM employees-00001,other-employees-*
 ----
 
-Use the `METADATA` directive to enable <<esql-metadata-fields,metadata fields>>:
+Use the optional `METADATA` directive to enable <<esql-metadata-fields,metadata fields>>:
 
 [source,esql]
 ----
-FROM employees [METADATA _id]
+FROM employees METADATA _id
 ----

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java

@@ -161,16 +161,18 @@ public class MultiClusterSpecIT extends EsqlSpecTestCase {
         String query = testCase.query;
         String[] commands = query.split("\\|");
         String first = commands[0].trim();
+
         if (commands[0].toLowerCase(Locale.ROOT).startsWith("from")) {
-            String[] parts = commands[0].split("\\[");
+            String[] parts = commands[0].split("(?i)metadata");
             assert parts.length >= 1 : parts;
             String fromStatement = parts[0];
+
             String[] localIndices = fromStatement.substring("FROM ".length()).split(",");
             String remoteIndices = Arrays.stream(localIndices)
                 .map(index -> "*:" + index.trim() + "," + index.trim())
                 .collect(Collectors.joining(","));
-            var newFrom = "FROM " + remoteIndices + commands[0].substring(fromStatement.length());
-            testCase.query = newFrom + " " + query.substring(first.length());
+            var newFrom = "FROM " + remoteIndices + " " + commands[0].substring(fromStatement.length());
+            testCase.query = newFrom + query.substring(first.length());
         }
         int offset = testCase.query.length() - query.length();
         if (offset != 0) {
@@ -195,7 +197,7 @@ public class MultiClusterSpecIT extends EsqlSpecTestCase {
     static boolean hasIndexMetadata(String query) {
         String[] commands = query.split("\\|");
         if (commands[0].trim().toLowerCase(Locale.ROOT).startsWith("from")) {
-            String[] parts = commands[0].split("\\[");
+            String[] parts = commands[0].split("(?i)metadata");
             return parts.length > 1 && parts[1].contains("_index");
         }
         return false;

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java

@@ -441,7 +441,7 @@ public abstract class RestEsqlTestCase extends ESRestTestCase {
         request.setJsonEntity("{\"a\": 3}");
         assertEquals(201, client().performRequest(request).getStatusLine().getStatusCode());
 
-        var query = fromIndex() + "* [metadata _index, _version, _id] | sort _version";
+        var query = fromIndex() + "* metadata _index, _version, _id | sort _version";
         Map<String, Object> result = runEsql(new RequestObjectBuilder().query(query));
         var columns = List.of(
             Map.of("name", "a", "type", "long"),

x-pack/plugin/esql/qa/testFixtures/src/main/resources/id.csv-spec

@@ -1,9 +1,8 @@
 //
-// Tests for _id fields
 //
 
-selectAll
-FROM apps [metadata _id];
+selectAll#[skip:-8.12.99]
+FROM apps metadata _id;
 ignoreOrder:true
 
 id:integer |name:keyword   |version:version     | _id:keyword
@@ -23,22 +22,23 @@ id:integer |name:keyword   |version:version     | _id:keyword
 14         |mmmmm          |5.2.9               | 14
 ;
 
-filterById
-FROM apps [metadata _id]| WHERE _id == "4";
+filterById#[skip:-8.12.99]
+FROM apps metadata _id | WHERE _id == "4";
 
 id:i |name:k |version:v | _id:k
 4    |ddddd  |2.12.0    | 4
 ;
 
-keepId
-FROM apps [metadata _id] | WHERE id == 3 | KEEP _id;
+keepId#[skip:-8.12.99]
+FROM apps metadata _id | WHERE id == 3 | KEEP _id;
+
 
 _id:k
 3
 ;
 
-idRangeAndSort
-FROM apps [metadata _id] | WHERE _id >= "2" AND _id <= "7" | SORT _id | keep id, name, _id;
+idRangeAndSort#[skip:-8.12.99]
+FROM apps metadata _id | WHERE _id >= "2" AND _id <= "7" | SORT _id | keep id, name, _id;
 
 id:i    |name:k       | _id:k
 2       |bbbbb        | 2
@@ -49,8 +49,8 @@ id:i    |name:k       | _id:k
 7       |ggggg        | 7
 ;
 
-orderById
-FROM apps [metadata _id] | KEEP _id, name | SORT _id;
+orderById#[skip:-8.12.99]
+FROM apps metadata _id | KEEP _id, name | SORT _id;
 
 _id:k  | name:s
 1      | aaaaa
@@ -69,8 +69,8 @@ _id:k  | name:s
 9      | iiiii
 ;
 
-orderByIdDesc
-FROM apps [metadata _id] | KEEP _id, name | SORT _id DESC;
+orderByIdDesc#[skip:-8.12.99]
+FROM apps metadata _id | KEEP _id, name | SORT _id DESC;
 
 _id:k  | name:s
 
@@ -90,8 +90,8 @@ _id:k  | name:s
 1      | aaaaa
 ;
 
-concatId
-FROM apps [metadata _id] | eval c = concat(_id, name) | SORT _id | KEEP c;
+concatId#[skip:-8.12.99]
+FROM apps metadata _id | eval c = concat(_id, name) | SORT _id | KEEP c;
 
 c:k
 1aaaaa
@@ -110,16 +110,16 @@ c:k
 9iiiii
 ;
 
-statsOnId
-FROM apps [metadata _id] | stats c = count(_id), d = count_distinct(_id);
+statsOnId#[skip:-8.12.99]
+FROM apps metadata _id | stats c = count(_id), d = count_distinct(_id);
 
 c:l | d:l
 14  | 14
 ;
 
 
-statsOnIdByGroup
-FROM apps [metadata _id] | stats c = count(_id) by name | sort c desc, name | limit 5;
+statsOnIdByGroup#[skip:-8.12.99]
+FROM apps metadata _id | stats c = count(_id) by name | sort c desc, name | limit 5;
 
 c:l | name:k
 2   | aaaaa

x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata-IT_tests_only.csv-spec

@@ -1,31 +1,32 @@
 
-simpleKeep
-from employees [metadata _index, _version] | sort emp_no | limit 2 | keep emp_no, _index, _version;
+simpleKeep#[skip:-8.12.99]
+from employees metadata _index, _version | sort emp_no | limit 2 | keep emp_no, _index, _version;
 
 emp_no:integer |_index:keyword |_version:long
 10001          |employees      |1
 10002          |employees      |1
 ;
 
-aliasWithSameName
-from employees [metadata _index, _version] | sort emp_no | limit 2 | eval _index = _index, _version = _version | keep emp_no, _index, _version;
+aliasWithSameName#[skip:-8.12.99]
+from employees metadata _index, _version | sort emp_no | limit 2 | eval _index = _index, _version = _version | keep emp_no, _index, _version;
 
 emp_no:integer |_index:keyword |_version:long
 10001          |employees      |1
 10002          |employees      |1
 ;
 
-inComparison
-from employees [metadata _index, _version] | sort emp_no | where _index == "employees" | where _version == 1 | keep emp_no | limit 2;
+inComparison#[skip:-8.12.99]
+from employees metadata _index, _version | sort emp_no | where _index == "employees" | where _version == 1 | keep emp_no | limit 2;
+
 
 emp_no:integer
 10001
 10002
 ;
 
-metaIndexInAggs
+metaIndexInAggs#[skip:-8.12.99]
 // tag::metaIndexInAggs[]
-FROM employees [METADATA _index, _id]
+FROM employees METADATA _index, _id
 | STATS max = MAX(emp_no) BY _index
 // end::metaIndexInAggs[]
 ;
@@ -36,73 +37,74 @@ max:integer |_index:keyword
 // end::metaIndexInAggs-result[]
 ;
 
-metaIndexAliasedInAggs
-from employees [metadata _index] | eval _i = _index | stats max = max(emp_no) by _i;
+metaIndexAliasedInAggs#[skip:-8.12.99]
+from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i;
+
 
 max:integer |_i:keyword
 10100       |employees
 ;
 
-metaVersionInAggs
-from employees [metadata _version] | stats min = min(emp_no) by _version;
+metaVersionInAggs#[skip:-8.12.99]
+from employees metadata _version | stats min = min(emp_no) by _version;
 
 min:integer |_version:long
 10001       |1
 ;
 
-metaVersionAliasedInAggs
-from employees [metadata _version] | eval _v = _version | stats min = min(emp_no) by _v;
+metaVersionAliasedInAggs#[skip:-8.12.99]
+from employees metadata _version | eval _v = _version | stats min = min(emp_no) by _v;
 
 min:integer |_v:long
 10001       |1
 ;
 
-inAggsAndAsGroups
-from employees [metadata _index, _version] | stats max = max(_version) by _index;
+inAggsAndAsGroups#[skip:-8.12.99]
+from employees metadata _index, _version | stats max = max(_version) by _index;
 
 max:long |_index:keyword
 1        |employees
 ;
 
-inAggsAndAsGroupsAliased
-from employees [metadata _index, _version] | eval _i = _index, _v = _version | stats max = max(_v) by _i;
+inAggsAndAsGroupsAliased#[skip:-8.12.99]
+from employees metadata _index, _version | eval _i = _index, _v = _version | stats max = max(_v) by _i;
 
 max:long |_i:keyword
 1        |employees
 ;
 
-inFunction
-from employees [metadata _index, _version] | sort emp_no | where length(_index) == length("employees") | where abs(_version) == 1 | keep emp_no | limit 2;
+inFunction#[skip:-8.12.99]
+from employees metadata _index, _version | sort emp_no | where length(_index) == length("employees") | where abs(_version) == 1 | keep emp_no | limit 2;
 
 emp_no:integer
 10001
 10002
 ;
 
-inArithmetics
-from employees [metadata _index, _version] | eval i = _version + 2 | stats min = min(emp_no) by i;
+inArithmetics#[skip:-8.12.99]
+from employees metadata _index, _version | eval i = _version + 2 | stats min = min(emp_no) by i;
 
 min:integer |i:long
 10001       |3
 ;
 
-inSort
-from employees [metadata _index, _version] | sort _version, _index, emp_no | keep emp_no, _version, _index | limit 2;
+inSort#[skip:-8.12.99]
+from employees metadata _index, _version | sort _version, _index, emp_no | keep emp_no, _version, _index | limit 2;
 
 emp_no:integer |_version:long |_index:keyword
 10001          |1             |employees
 10002          |1             |employees
 ;
 
-withMvFunction
-from employees [metadata _version] | eval i = mv_avg(_version) + 2 | stats min = min(emp_no) by i;
+withMvFunction#[skip:-8.12.99]
+from employees metadata _version | eval i = mv_avg(_version) + 2 | stats min = min(emp_no) by i;
 
 min:integer |i:double
 10001       |3.0
 ;
 
-overwritten
-from employees [metadata _index, _version] | sort emp_no | eval _index = 3, _version = "version" | keep emp_no, _index, _version | limit 3;
+overwritten#[skip:-8.12.99]
+from employees metadata _index, _version | sort emp_no | eval _index = 3, _version = "version" | keep emp_no, _index, _version | limit 3;
 
 emp_no:integer |_index:integer |_version:keyword
 10001          |3              |version
@@ -110,9 +112,9 @@ emp_no:integer |_index:integer |_version:keyword
 10003          |3              |version
 ;
 
-multipleIndices
+multipleIndices#[skip:-8.12.99]
 // tag::multipleIndices[]
-FROM ul_logs, apps [METADATA _index, _version]
+FROM ul_logs, apps METADATA _index, _version
 | WHERE id IN (13, 14) AND _version == 1
 | EVAL key = CONCAT(_index, "_", TO_STR(id))
 | SORT id, _index

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java

@@ -114,7 +114,7 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase {
     }
 
     public void testMetadataIndex() {
-        try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* [METADATA _index] | stats sum(v) by _index | sort _index")) {
+        try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* METADATA _index | stats sum(v) by _index | sort _index")) {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values.get(0), equalTo(List.of(285L, "cluster-a:logs-2")));
             assertThat(values.get(1), equalTo(List.of(45L, "logs-1")));

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java

@@ -1222,7 +1222,7 @@ public class EsqlActionIT extends AbstractEsqlIntegTestCase {
     }
 
     public void testLoadId() {
-        try (EsqlQueryResponse results = run("from test [metadata _id] | keep _id | sort _id ")) {
+        try (EsqlQueryResponse results = run("from test metadata _id | keep _id | sort _id ")) {
             assertThat(results.columns(), equalTo(List.of(new ColumnInfo("_id", "keyword"))));
             ListMatcher values = matchesList();
             for (int i = 10; i < 50; i++) {
@@ -1427,7 +1427,7 @@ public class EsqlActionIT extends AbstractEsqlIntegTestCase {
 
         assertEmptyIndexQueries(from);
 
-        try (EsqlQueryResponse resp = run(from + "[METADATA _source] | EVAL x = 123")) {
+        try (EsqlQueryResponse resp = run(from + "METADATA _source | EVAL x = 123")) {
             assertFalse(resp.values().hasNext());
             assertThat(resp.columns(), equalTo(List.of(new ColumnInfo("_source", "_source"), new ColumnInfo("x", "integer"))));
         }
@@ -1455,7 +1455,7 @@ public class EsqlActionIT extends AbstractEsqlIntegTestCase {
 
         assertEmptyIndexQueries(from);
 
-        try (EsqlQueryResponse resp = run(from + "[METADATA _source] | EVAL x = 123")) {
+        try (EsqlQueryResponse resp = run(from + "METADATA _source | EVAL x = 123")) {
             assertFalse(resp.values().hasNext());
             assertThat(
                 resp.columns(),
@@ -1470,7 +1470,7 @@ public class EsqlActionIT extends AbstractEsqlIntegTestCase {
     }
 
     private void assertEmptyIndexQueries(String from) {
-        try (EsqlQueryResponse resp = run(from + "[METADATA _source] | KEEP _source | LIMIT 1")) {
+        try (EsqlQueryResponse resp = run(from + "METADATA _source | KEEP _source | LIMIT 1")) {
             assertFalse(resp.values().hasNext());
             assertThat(resp.columns(), equalTo(List.of(new ColumnInfo("_source", "_source"))));
         }

x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4

@@ -102,9 +102,17 @@ fromCommand
     ;
 
 metadata
-    : OPENING_BRACKET METADATA fromIdentifier (COMMA fromIdentifier)* CLOSING_BRACKET
+    : metadataOption
+    | deprecated_metadata
     ;
 
+metadataOption
+    : METADATA fromIdentifier (COMMA fromIdentifier)*
+    ;
+
+deprecated_metadata
+    : OPENING_BRACKET metadataOption CLOSING_BRACKET
+    ;
 
 evalCommand
     : EVAL fields

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java

@@ -348,6 +348,30 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitMetadata(EsqlBaseParser.MetadataContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { }
   /**
    * {@inheritDoc}
    *

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java

@@ -208,6 +208,20 @@ public class EsqlBaseParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitMetadata(EsqlBaseParser.MetadataContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java

@@ -321,6 +321,26 @@ public interface EsqlBaseParserListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitMetadata(EsqlBaseParser.MetadataContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#metadataOption}.
+   * @param ctx the parse tree
+   */
+  void enterMetadataOption(EsqlBaseParser.MetadataOptionContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#metadataOption}.
+   * @param ctx the parse tree
+   */
+  void exitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}.
+   * @param ctx the parse tree
+   */
+  void enterDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}.
+   * @param ctx the parse tree
+   */
+  void exitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx);
   /**
    * Enter a parse tree produced by {@link EsqlBaseParser#evalCommand}.
    * @param ctx the parse tree

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java

@@ -195,6 +195,18 @@ public interface EsqlBaseParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitMetadata(EsqlBaseParser.MetadataContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#metadataOption}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx);
   /**
    * Visit a parse tree produced by {@link EsqlBaseParser#evalCommand}.
    * @param ctx the parse tree

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java

@@ -12,6 +12,7 @@ import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.elasticsearch.dissect.DissectException;
 import org.elasticsearch.dissect.DissectParser;
+import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.MetadataOptionContext;
 import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.QualifiedNamePatternContext;
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
 import org.elasticsearch.xpack.esql.plan.logical.Drop;
@@ -60,6 +61,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
 
+import static org.elasticsearch.common.logging.HeaderWarning.addWarning;
 import static org.elasticsearch.xpack.esql.plan.logical.Enrich.Mode;
 import static org.elasticsearch.xpack.ql.parser.ParserUtils.source;
 import static org.elasticsearch.xpack.ql.parser.ParserUtils.typedParsing;
@@ -180,7 +182,21 @@ public class LogicalPlanBuilder extends ExpressionBuilder {
         TableIdentifier table = new TableIdentifier(source, null, visitFromIdentifiers(ctx.fromIdentifier()));
         Map<String, Attribute> metadataMap = new LinkedHashMap<>();
         if (ctx.metadata() != null) {
-            for (var c : ctx.metadata().fromIdentifier()) {
+            var deprecatedContext = ctx.metadata().deprecated_metadata();
+            MetadataOptionContext metadataOptionContext = null;
+            if (deprecatedContext != null) {
+                var s = source(deprecatedContext).source();
+                addWarning(
+                    "Line {}:{}: Square brackets '[]' need to be removed in FROM METADATA declaration",
+                    s.getLineNumber(),
+                    s.getColumnNumber()
+                );
+                metadataOptionContext = deprecatedContext.metadataOption();
+            } else {
+                metadataOptionContext = ctx.metadata().metadataOption();
+
+            }
+            for (var c : metadataOptionContext.fromIdentifier()) {
                 String id = visitFromIdentifier(c);
                 Source src = source(c);
                 if (MetadataAttribute.isSupported(id) == false) {

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java

@@ -3219,11 +3219,11 @@ public class LogicalPlanOptimizerTests extends ESTestCase {
         as(plan, LocalRelation.class);
         assertThat(plan.output(), equalTo(NO_FIELDS));
 
-        plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test [metadata _id] | eval x = 1")));
+        plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test metadata _id | eval x = 1")));
         as(plan, LocalRelation.class);
         assertThat(Expressions.names(plan.output()), contains("_id", "x"));
 
-        plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test [metadata _id, _version] | limit 5")));
+        plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test metadata _id, _version | limit 5")));
         as(plan, LocalRelation.class);
         assertThat(Expressions.names(plan.output()), contains("_id", "_version"));
 

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java

@@ -1739,7 +1739,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
 
     public void testPushDownMetadataIndexInWildcard() {
         var plan = physicalPlan("""
-            from test [metadata _index]
+            from test metadata _index
             | where _index like "test*"
             """);
 
@@ -1766,7 +1766,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
      */
     public void testPushDownMetadataIndexInEquality() {
         var plan = physicalPlan("""
-            from test [metadata _index]
+            from test metadata _index
             | where _index == "test"
             """);
 
@@ -1793,7 +1793,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
      */
     public void testPushDownMetadataIndexInNotEquality() {
         var plan = physicalPlan("""
-            from test [metadata _index]
+            from test metadata _index
             | where _index != "test"
             """);
 
@@ -1830,7 +1830,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
             tuple("<=", LessThanOrEqual.class)
             // no NullEquals use
         )) {
-            var plan = physicalPlan("from test [metadata _index] | where _index " + t.v1() + " \"test\"");
+            var plan = physicalPlan("from test metadata _index | where _index " + t.v1() + " \"test\"");
 
             var optimized = optimizedPlan(plan);
             var limit = as(optimized, LimitExec.class);
@@ -1851,7 +1851,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
 
     public void testDontPushDownMetadataVersionAndId() {
         for (var t : List.of(tuple("_version", "2"), tuple("_id", "\"2\""))) {
-            var plan = physicalPlan("from test [metadata " + t.v1() + "] | where " + t.v1() + " == " + t.v2());
+            var plan = physicalPlan("from test metadata " + t.v1() + " | where " + t.v1() + " == " + t.v2());
 
             var optimized = optimizedPlan(plan);
             var limit = as(optimized, LimitExec.class);

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java

@@ -580,32 +580,32 @@ public class StatementParserTests extends ESTestCase {
     }
 
     public void testMetadataFieldOnOtherSources() {
+        expectError("row a = 1 metadata _index", "line 1:20: extraneous input '_index' expecting <EOF>");
+        expectError("show functions metadata _index", "line 1:16: token recognition error at: 'm'");
         expectError(
-            "row a = 1 [metadata _index]",
-            "1:11: mismatched input '[' expecting {<EOF>, '|', 'and', ',', 'or', '+', '-', '*', '/', '%'}"
+            "explain [from foo] metadata _index",
+            "line 1:20: mismatched input 'metadata' expecting {'|', ',', OPENING_BRACKET, ']', 'metadata'}"
         );
-        expectError("show functions [metadata _index]", "line 1:16: token recognition error at: '['");
-        expectError("explain [from foo] [metadata _index]", "line 1:20: mismatched input '[' expecting {'|', ',', OPENING_BRACKET, ']'}");
     }
 
     public void testMetadataFieldMultipleDeclarations() {
-        expectError("from test [metadata _index, _version, _index]", "1:40: metadata field [_index] already declared [@1:21]");
+        expectError("from test metadata _index, _version, _index", "1:39: metadata field [_index] already declared [@1:20]");
     }
 
     public void testMetadataFieldUnsupportedPrimitiveType() {
-        expectError("from test [metadata _tier]", "line 1:22: unsupported metadata field [_tier]");
+        expectError("from test metadata _tier", "line 1:21: unsupported metadata field [_tier]");
     }
 
     public void testMetadataFieldUnsupportedCustomType() {
-        expectError("from test [metadata _feature]", "line 1:22: unsupported metadata field [_feature]");
+        expectError("from test metadata _feature", "line 1:21: unsupported metadata field [_feature]");
     }
 
     public void testMetadataFieldNotFoundNonExistent() {
-        expectError("from test [metadata _doesnot_compute]", "line 1:22: unsupported metadata field [_doesnot_compute]");
+        expectError("from test metadata _doesnot_compute", "line 1:21: unsupported metadata field [_doesnot_compute]");
     }
 
     public void testMetadataFieldNotFoundNormalField() {
-        expectError("from test [metadata emp_no]", "line 1:22: unsupported metadata field [emp_no]");
+        expectError("from test metadata emp_no", "line 1:21: unsupported metadata field [emp_no]");
     }
 
     public void testDissectPattern() {

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java

@@ -26,7 +26,7 @@ public class IndexResolverFieldNamesTests extends ESTestCase {
     }
 
     public void testBasicFromCommandWithMetadata() {
-        assertFieldNames("from test [metadata _index, _id, _version]", ALL_FIELDS);
+        assertFieldNames("from test metadata _index, _id, _version", ALL_FIELDS);
     }
 
     public void testBasicEvalAndDrop() {
@@ -620,43 +620,43 @@ public class IndexResolverFieldNamesTests extends ESTestCase {
     }
 
     public void testSelectAll() {
-        assertFieldNames("FROM apps [metadata _id]", ALL_FIELDS);
+        assertFieldNames("FROM apps metadata _id", ALL_FIELDS);
     }
 
     public void testFilterById() {
-        assertFieldNames("FROM apps [metadata _id]| WHERE _id == \"4\"", ALL_FIELDS);
+        assertFieldNames("FROM apps metadata _id| WHERE _id == \"4\"", ALL_FIELDS);
     }
 
     public void testKeepId() {
-        assertFieldNames("FROM apps [metadata _id] | WHERE id == 3 | KEEP _id", Set.of("id", "id.*"));
+        assertFieldNames("FROM apps metadata _id | WHERE id == 3 | KEEP _id", Set.of("id", "id.*"));
     }
 
     public void testIdRangeAndSort() {
         assertFieldNames("""
-            FROM apps [metadata _id]
+            FROM apps metadata _id
             | WHERE _id >= "2" AND _id <= "7"
             | SORT _id
             | keep id, name, _id""", Set.of("id", "id.*", "name", "name.*"));
     }
 
     public void testOrderById() {
-        assertFieldNames("FROM apps [metadata _id] | KEEP _id, name | SORT _id", Set.of("name", "name.*"));
+        assertFieldNames("FROM apps metadata _id | KEEP _id, name | SORT _id", Set.of("name", "name.*"));
     }
 
     public void testOrderByIdDesc() {
-        assertFieldNames("FROM apps [metadata _id] | KEEP _id, name | SORT _id DESC", Set.of("name", "name.*"));
+        assertFieldNames("FROM apps metadata _id | KEEP _id, name | SORT _id DESC", Set.of("name", "name.*"));
     }
 
     public void testConcatId() {
-        assertFieldNames("FROM apps [metadata _id] | eval c = concat(_id, name) | SORT _id | KEEP c", Set.of("name", "name.*"));
+        assertFieldNames("FROM apps metadata _id | eval c = concat(_id, name) | SORT _id | KEEP c", Set.of("name", "name.*"));
     }
 
     public void testStatsOnId() {
-        assertFieldNames("FROM apps [metadata _id] | stats c = count(_id), d = count_distinct(_id)", INDEX_METADATA_FIELD);
+        assertFieldNames("FROM apps metadata _id | stats c = count(_id), d = count_distinct(_id)", INDEX_METADATA_FIELD);
     }
 
     public void testStatsOnIdByGroup() {
-        assertFieldNames("FROM apps [metadata _id] | stats c = count(_id) by name | sort c desc, name | limit 5", Set.of("name", "name.*"));
+        assertFieldNames("FROM apps metadata _id | stats c = count(_id) by name | sort c desc, name | limit 5", Set.of("name", "name.*"));
     }
 
     public void testSimpleProject() {
@@ -709,10 +709,7 @@ public class IndexResolverFieldNamesTests extends ESTestCase {
     }
 
     public void testMetaIndexAliasedInAggs() {
-        assertFieldNames(
-            "from employees [metadata _index] | eval _i = _index | stats max = max(emp_no) by _i",
-            Set.of("emp_no", "emp_no.*")
-        );
+        assertFieldNames("from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i", Set.of("emp_no", "emp_no.*"));
     }
 
     public void testCoalesceFolding() {

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml

@@ -58,8 +58,8 @@
 ---
 "unsupported and invalid mapped fields":
   - skip:
-      version: " - 8.11.99"
-      reason: "fixes in 8.12 or later"
+      version: " - 8.12.99"
+      reason: "fixes in 8.13 or later"
   - do:
       indices.create:
         index: index1
@@ -107,7 +107,7 @@
   - do:
       esql.query:
         body:
-          query: 'from index* [metadata _index] | limit 5 | sort _index desc'
+          query: 'from index* metadata _index | limit 5 | sort _index desc'
   - match: { columns.0.name: http.headers }
   - match: { columns.0.type: unsupported }
   - match: { columns.1.name: http.headers.location }

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml

@@ -621,6 +621,9 @@ version:
 
 ---
 id:
+  - skip:
+      version: " - 8.12.99"
+      reason: "_source is available in 8.13+"
   - do:
       indices.create:
         index: test
@@ -643,7 +646,7 @@ id:
         - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
-          query: 'from test [metadata _id] | keep _id, kw'
+          query: 'from test metadata _id | keep _id, kw'
   - match: { columns.0.name: _id }
   - match: { columns.0.type: keyword }
   - length: { values: 1 }
@@ -683,8 +686,8 @@ unsigned_long:
 ---
 _source:
   - skip:
-      version: " - 8.11.99"
-      reason: "_source is available in 8.12+"
+      version: " - 8.12.99"
+      reason: "_source is available in 8.13+"
 
   - do:
       bulk:
@@ -697,7 +700,7 @@ _source:
   - do:
       esql.query:
         body:
-          query: 'FROM test [METADATA _source] | KEEP _source | LIMIT 1'
+          query: 'FROM test METADATA _source | KEEP _source | LIMIT 1'
   - match: { columns.0.name: _source }
   - match: { columns.0.type: _source }
   - length: { values: 1 }
@@ -711,8 +714,8 @@ _source:
 ---
 _source keep all:
   - skip:
-      version: " - 8.11.99"
-      reason: "_source is available in 8.12+"
+      version: " - 8.12.99"
+      reason: "_source is available in 8.13+"
 
   - do:
       indices.create:
@@ -732,7 +735,7 @@ _source keep all:
   - do:
       esql.query:
         body:
-          query: 'FROM test [METADATA _source] | LIMIT 1'
+          query: 'FROM test METADATA _source | LIMIT 1'
   - match: { columns.0.name: _source }
   - match: { columns.0.type: _source }
   - length: { values: 1 }
@@ -746,8 +749,8 @@ _source keep all:
 ---
 _source disabled:
   - skip:
-      version: " - 8.11.99"
-      reason: "_source is available in 8.12+"
+      version: " - 8.12.99"
+      reason: "_source is available in 8.13+"
 
   - do:
       indices.create:
@@ -768,7 +771,7 @@ _source disabled:
   - do:
       esql.query:
         body:
-          query: 'FROM test [METADATA _source] | KEEP _source | LIMIT 1'
+          query: 'FROM test METADATA _source | KEEP _source | LIMIT 1'
   - match: { columns.0.name: _source }
   - match: { columns.0.type: _source }
   - length: { values: 1 }

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml

@@ -107,13 +107,16 @@ setup:
 
 ---
 load everything:
+  - skip:
+      version: " - 8.12.99"
+      reason: "_source is available in 8.13+"
   - do:
       allowed_warnings_regex:
         - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null"
         - "No limit defined, adding default limit of \\[.*\\]"
       esql.query:
         body:
-          query: 'from test [metadata _id]'
+          query: 'from test metadata _id'
 
   - match: {columns.0.name: "@timestamp"}
   - match: {columns.0.type: "date"}
@@ -231,8 +234,8 @@ from index pattern explicit counter use:
 ---
 _source:
   - skip:
-      version: " - 8.11.99"
-      reason: "_source is available in 8.12+"
+      version: " - 8.12.99"
+      reason: "_source is available in 8.13+"
 
   - do:
       bulk:
@@ -245,7 +248,7 @@ _source:
   - do:
       esql.query:
         body:
-          query: 'FROM test [METADATA _source] | WHERE @timestamp == "2021-04-28T18:50:23.142Z" | KEEP _source | LIMIT 1'
+          query: 'FROM test METADATA _source | WHERE @timestamp == "2021-04-28T18:50:23.142Z" | KEEP _source | LIMIT 1'
   - match: { columns.0.name: _source }
   - match: { columns.0.type: _source }
   - length: { values: 1 }

x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml

@@ -87,8 +87,12 @@ teardown:
 
 ---
 "Index data and search on the mixed cluster":
+  - skip:
+      features: allowed_warnings
 
   - do:
+      allowed_warnings:
+        - "Line 1:21: Square brackets '[]' need to be removed in FROM METADATA declaration"
       headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" }
       esql.query:
         body:
@@ -111,6 +115,8 @@ teardown:
   - match: {values.4.1: "tablet" }
 
   - do:
+      allowed_warnings:
+        - "Line 1:21: Square brackets '[]' need to be removed in FROM METADATA declaration"
       headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" }
       esql.query:
         body: