[8.x] ESQL: Support for _index metadata field in CsvTests (#121261) (#121499)

Manually cherry picking #121261.

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java

@@ -32,6 +32,7 @@ public class MetadataAttribute extends TypedAttribute {
     public static final String TIMESTAMP_FIELD = "@timestamp";
     public static final String TSID_FIELD = "_tsid";
     public static final String SCORE = "_score";
+    public static final String INDEX = "_index";
 
     static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         Attribute.class,
@@ -42,7 +43,7 @@ public class MetadataAttribute extends TypedAttribute {
     private static final Map<String, Tuple<DataType, Boolean>> ATTRIBUTES_MAP = Map.of(
         "_version",
         tuple(DataType.LONG, false), // _version field is not searchable
-        "_index",
+        INDEX,
         tuple(DataType.KEYWORD, true),
         IdFieldMapper.NAME,
         tuple(DataType.KEYWORD, false), // actually searchable, but fielddata access on the _id field is disallowed by default

x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata-remote.csv-spec

@@ -39,7 +39,7 @@ max:integer |_index:keyword
 ;
 
 metaIndexAliasedInAggs
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: metadata_fields_remote_test
 from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i | SORT _i;
 

x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata.csv-spec

@@ -40,7 +40,7 @@ max:integer |_index:keyword
 ;
 
 metaIndexSorted
-required_capability: metadata_fields
+required_capability: index_metadata_field
 from employees metadata _index | sort _index, emp_no desc | keep emp_no, _index | limit 2;
 
 
@@ -50,7 +50,7 @@ emp_no:integer |_index:keyword
 ;
 
 metaIndexWithInPredicate
-required_capability: metadata_fields
+required_capability: index_metadata_field
 from employees metadata _index | where _index in ("employees", "foobar") | sort emp_no desc | keep emp_no, _index | limit 2;
 
 
@@ -60,7 +60,7 @@ emp_no:integer |_index:keyword
 ;
 
 metaIndexAliasedInAggs
-required_capability: metadata_fields
+required_capability: index_metadata_field
 from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i;
 
 

x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec

@@ -133,7 +133,7 @@ mc:l | count:l
 
 multiIndexIpString
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: casting_operator
 required_capability: union_types_remove_fields
 
@@ -162,7 +162,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233
 
 multiIndexIpStringRename
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: casting_operator
 required_capability: union_types_remove_fields
 
@@ -191,7 +191,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233
 
 multiIndexIpStringRenameToString
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_str METADATA _index
@@ -219,7 +219,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z  |  172.21.2.162     |  3450233
 
 multiIndexWhereIpString
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_str METADATA _index
@@ -237,7 +237,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z  |  3450233              |  Connected
 
 multiIndexWhereIpStringLike
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_str METADATA _index
@@ -445,7 +445,7 @@ count:long  |  message:keyword
 
 multiIndexMissingIpToString
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_missing_field
 
 FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
@@ -480,7 +480,7 @@ sample_data_str        | 2023-10-23T12:15:03.360Z  |  172.21.2.162       |  3450
 
 multiIndexMissingIpToIp
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_missing_field
 
 FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
@@ -515,7 +515,7 @@ sample_data_str        | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233
 
 multiIndexTsLong
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_ts_long METADATA _index
@@ -543,7 +543,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233
 
 multiIndexTsLongRename
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_ts_long METADATA _index
@@ -573,7 +573,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233
 multiIndexTsNanosRename
 required_capability: to_date_nanos
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_ts_nanos METADATA _index
@@ -602,7 +602,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233
 multiIndexTsNanosRenameToNanos
 required_capability: to_date_nanos
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_ts_nanos METADATA _index
@@ -631,7 +631,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z  |  172.21.2.162  |  34502
 multiIndex sort millis and nanos as nanos
 required_capability: to_date_nanos
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_ts_nanos METADATA _index
@@ -660,7 +660,7 @@ sample_data          | 2023-10-23T12:15:03.360000000Z  |  172.21.2.162  |  34502
 multiIndex sort millis and nanos as millis
 required_capability: to_date_nanos
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_ts_nanos METADATA _index
@@ -691,7 +691,7 @@ multiIndexTsNanosRenameToNanosWithFiltering
 required_capability: to_date_nanos
 required_capability: date_nanos_binary_comparison
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_ts_nanos METADATA _index
@@ -716,7 +716,7 @@ sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z  |  172.21.0.5    |  12323
 
 multiIndexTsLongRenameToString
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_ts_long METADATA _index
@@ -744,7 +744,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233
 
 multiIndexWhereTsLong
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 
 FROM sample_data, sample_data_ts_long METADATA _index
@@ -979,7 +979,7 @@ count:long  |  message:keyword
 
 multiIndexIpStringTsLong
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 required_capability: to_date_nanos
 
@@ -1022,7 +1022,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233
 
 multiIndexIpStringTsLongDropped
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: to_date_nanos
 
 FROM sample_data* METADATA _index
@@ -1064,7 +1064,7 @@ sample_data_ts_nanos |  8268153              |  Connection error
 
 multiIndexIpStringTsLongRename
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 required_capability: to_date_nanos
 
@@ -1107,7 +1107,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233
 
 multiIndexIpStringTsLongRenameDropped
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: to_date_nanos
 
 FROM sample_data* METADATA _index
@@ -1149,7 +1149,7 @@ sample_data_ts_nanos |  8268153              |  Connection error
 
 multiIndexIpStringTsLongRenameToString
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 required_capability: to_date_nanos
 
@@ -1192,7 +1192,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z  |  172.21.2.162     |  3450233
 
 multiIndexWhereIpStringTsLong
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 required_capability: to_date_nanos
 
@@ -1226,7 +1226,7 @@ count:long  |  message:keyword
 
 multiIndexWhereIpStringLikeTsLong
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 required_capability: to_date_nanos
 
@@ -1260,7 +1260,7 @@ count:long  |  message:keyword
 
 multiIndexMultiColumnTypesRename
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 required_capability: to_date_nanos
 
@@ -1279,7 +1279,7 @@ null                   | null                  |  8268153            | Connectio
 
 multiIndexMultiColumnTypesRenameAndKeep
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 required_capability: to_date_nanos
 
@@ -1299,7 +1299,7 @@ sample_data_ts_nanos | 2023-10-23T13:52:55.015Z  | 2023-10-23T13:52:55.015123456
 
 multiIndexMultiColumnTypesRenameAndDrop
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: union_types_remove_fields
 required_capability: to_date_nanos
 
@@ -1591,7 +1591,7 @@ FROM sample_data, sample_data_ts_long
 
 shortIntegerWidening
 required_capability: union_types
-required_capability: metadata_fields
+required_capability: index_metadata_field
 required_capability: casting_operator
 required_capability: union_types_numeric_widening
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -564,6 +564,13 @@ public class EsqlCapabilities {
 
         /** Capability for remote metadata test */
         METADATA_FIELDS_REMOTE_TEST(false),
+
+        /**
+         * Support specifically for *just* the _index METADATA field. Used by CsvTests, since that is the only metadata field currently
+         * supported.
+         */
+        INDEX_METADATA_FIELD,
+
         /**
          * WIP on Join planning
          * - Introduce BinaryPlan and co

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java

@@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction;
+import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DateEsField;
 import org.elasticsearch.xpack.esql.core.type.EsField;
@@ -50,7 +51,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
 
 public class IndexResolver {
     public static final Set<String> ALL_FIELDS = Set.of("*");
-    public static final Set<String> INDEX_METADATA_FIELD = Set.of("_index");
+    public static final Set<String> INDEX_METADATA_FIELD = Set.of(MetadataAttribute.INDEX);
     public static final String UNMAPPED = "unmapped";
 
     public static final IndicesOptions FIELD_CAPS_INDICES_OPTIONS = IndicesOptions.builder()

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java

@@ -42,6 +42,7 @@ import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
 import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
@@ -289,6 +290,10 @@ public class TestPhysicalOperationProviders extends AbstractPhysicalOperationPro
     private Block extractBlockForSingleDoc(DocBlock docBlock, String columnName, TestBlockCopier blockCopier) {
         var indexId = docBlock.asVector().shards().getInt(0);
         var indexPage = indexPages.get(indexId);
+        if (MetadataAttribute.INDEX.equals(columnName)) {
+            return docBlock.blockFactory()
+                .newConstantBytesRefBlockWith(new BytesRef(indexPage.index), blockCopier.docIndices.getPositionCount());
+        }
         int columnIndex = indexPage.columnIndex(columnName)
             .orElseThrow(() -> new EsqlIllegalArgumentException("Cannot find column named [{}] in {}", columnName, indexPage.columnNames));
         var originalData = indexPage.page.getBlock(columnIndex);
@@ -408,8 +413,9 @@ public class TestPhysicalOperationProviders extends AbstractPhysicalOperationPro
                 TestBlockCopier blockCopier = mapToDocValues
                     ? TestSpatialPointStatsBlockCopier.create(indexDoc.asVector().docs(), dataType)
                     : new TestBlockCopier(indexDoc.asVector().docs());
-                Block blockForIndex = extractBlock.apply(indexDoc, blockCopier);
-                blockBuilder.copyFrom(blockForIndex, 0, blockForIndex.getPositionCount());
+                try (Block blockForIndex = extractBlock.apply(indexDoc, blockCopier)) {
+                    blockBuilder.copyFrom(blockForIndex, 0, blockForIndex.getPositionCount());
+                }
             });
             var result = blockBuilder.build();
             assert result.getPositionCount() == docBlock.getPositionCount()