ESQL: Improve syntax for LOOKUP tables (#109489)

Replace the syntax for `tables` with something a little more natural.

Now it is:

```
$ curl -uelastic:password -HContent-Type:application/json -XPOST \
    'localhost:9200/_query?error_trace&pretty&format=txt' \
-d'{
    "query": "ROW a=1::LONG | LOOKUP t ON a",
    "tables": {
        "t": {
            "a": {"long":     [    1,     4,     2]},
            "v1": {"integer": [   10,    11,    12]},
            "v2": {"keyword": ["cat", "dog", "wow"]}
        }
    }
}'
      v1       |      v2       |       a
---------------+---------------+---------------
10             |cat            |1
```

docs/reference/esql/processing-commands/lookup.asciidoc

@@ -40,8 +40,8 @@ POST /_query?format=txt
   """,
   "tables": {
     "era": {
-      "author:keyword": ["Frank Herbert", "Peter F. Hamilton", "Vernor Vinge", "Alastair Reynolds", "James S.A. Corey"],
-      "era:keyword"   : [ "The New Wave",           "Diamond",      "Diamond",           "Diamond",           "Hadron"]
+      "author": {"keyword": ["Frank Herbert", "Peter F. Hamilton", "Vernor Vinge", "Alastair Reynolds", "James S.A. Corey"]},
+      "era":    {"keyword": [ "The New Wave",           "Diamond",      "Diamond",           "Diamond",           "Hadron"]}
     }
   }
 }

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java

@@ -256,6 +256,23 @@ public class RestEsqlIT extends RestEsqlTestCase {
         assertThat(deleteIndex("index2").isAcknowledged(), Matchers.is(true));
     }
 
+    public void testTableDuplicateNames() throws IOException {
+        Request request = new Request("POST", "/_query");
+        request.setJsonEntity("""
+            {
+              "query": "FROM a=1",
+              "tables": {
+                "t": {
+                  "a": {"integer": [1]},
+                  "a": {"integer": [1]}
+                }
+              }
+            }""");
+        ResponseException re = expectThrows(ResponseException.class, () -> client().performRequest(request));
+        assertThat(re.getResponse().getStatusLine().getStatusCode(), equalTo(400));
+        assertThat(re.getMessage(), containsString("[6:10] Duplicate field 'a'"));
+    }
+
     private void assertException(String query, String... errorMessages) throws IOException {
         ResponseException re = expectThrows(ResponseException.class, () -> runEsqlSync(requestObjectBuilder().query(query)));
         assertThat(re.getResponse().getStatusLine().getStatusCode(), equalTo(400));

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java

@@ -321,43 +321,52 @@ public abstract class EsqlSpecTestCase extends ESRestTestCase {
      * "tables" parameter sent if there is a LOOKUP in the request. If you
      * add to this, you must also add to {@link EsqlTestUtils#tables};
      */
-    private Map<String, Map<String, List<?>>> tables() {
-        Map<String, Map<String, List<?>>> tables = new TreeMap<>();
+    private Map<String, Map<String, RestEsqlTestCase.TypeAndValues>> tables() {
+        Map<String, Map<String, RestEsqlTestCase.TypeAndValues>> tables = new TreeMap<>();
         tables.put(
             "int_number_names",
             EsqlTestUtils.table(
-                Map.entry("int:integer", IntStream.range(0, 10).boxed().toList()),
-                Map.entry("name:keyword", IntStream.range(0, 10).mapToObj(EsqlTestUtils::numberName).toList())
+                Map.entry("int", new RestEsqlTestCase.TypeAndValues("integer", IntStream.range(0, 10).boxed().toList())),
+                Map.entry(
+                    "name",
+                    new RestEsqlTestCase.TypeAndValues("keyword", IntStream.range(0, 10).mapToObj(EsqlTestUtils::numberName).toList())
+                )
             )
         );
         tables.put(
             "long_number_names",
             EsqlTestUtils.table(
-                Map.entry("long:long", LongStream.range(0, 10).boxed().toList()),
-                Map.entry("name:keyword", IntStream.range(0, 10).mapToObj(EsqlTestUtils::numberName).toList())
+                Map.entry("long", new RestEsqlTestCase.TypeAndValues("long", LongStream.range(0, 10).boxed().toList())),
+                Map.entry(
+                    "name",
+                    new RestEsqlTestCase.TypeAndValues("keyword", IntStream.range(0, 10).mapToObj(EsqlTestUtils::numberName).toList())
+                )
             )
         );
         tables.put(
             "double_number_names",
             EsqlTestUtils.table(
-                Map.entry("double:double", List.of(2.03, 2.08)),
-                Map.entry("name:keyword", List.of("two point zero three", "two point zero eight"))
+                Map.entry("double", new RestEsqlTestCase.TypeAndValues("double", List.of(2.03, 2.08))),
+                Map.entry("name", new RestEsqlTestCase.TypeAndValues("keyword", List.of("two point zero three", "two point zero eight")))
             )
         );
         tables.put(
             "double_number_names_with_null",
             EsqlTestUtils.table(
-                Map.entry("double:double", List.of(2.03, 2.08, 0.0)),
-                Map.entry("name:keyword", Arrays.asList("two point zero three", "two point zero eight", null))
+                Map.entry("double", new RestEsqlTestCase.TypeAndValues("double", List.of(2.03, 2.08, 0.0))),
+                Map.entry(
+                    "name",
+                    new RestEsqlTestCase.TypeAndValues("keyword", Arrays.asList("two point zero three", "two point zero eight", null))
+                )
             )
         );
         tables.put(
             "big",
             EsqlTestUtils.table(
-                Map.entry("aa:keyword", List.of("foo", "bar", "baz", "foo")),
-                Map.entry("ab:keyword", List.of("zoo", "zop", "zoi", "foo")),
-                Map.entry("na:integer", List.of(1, 10, 100, 2)),
-                Map.entry("nb:integer", List.of(-1, -10, -100, -2))
+                Map.entry("aa", new RestEsqlTestCase.TypeAndValues("keyword", List.of("foo", "bar", "baz", "foo"))),
+                Map.entry("ab", new RestEsqlTestCase.TypeAndValues("keyword", List.of("zoo", "zop", "zoi", "foo"))),
+                Map.entry("na", new RestEsqlTestCase.TypeAndValues("integer", List.of(1, 10, 100, 2))),
+                Map.entry("nb", new RestEsqlTestCase.TypeAndValues("integer", List.of(-1, -10, -100, -2)))
             )
         );
         return tables;

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java

@@ -109,11 +109,13 @@ public abstract class RestEsqlTestCase extends ESRestTestCase {
         this.mode = mode;
     }
 
+    public record TypeAndValues(String type, List<?> values) {}
+
     public static class RequestObjectBuilder {
         private final XContentBuilder builder;
         private boolean isBuilt = false;
 
-        private Map<String, Map<String, List<?>>> tables;
+        private Map<String, Map<String, TypeAndValues>> tables;
 
         private Boolean keepOnCompletion = null;
 
@@ -131,7 +133,7 @@ public abstract class RestEsqlTestCase extends ESRestTestCase {
             return this;
         }
 
-        public RequestObjectBuilder tables(Map<String, Map<String, List<?>>> tables) {
+        public RequestObjectBuilder tables(Map<String, Map<String, TypeAndValues>> tables) {
             this.tables = tables;
             return this;
         }
@@ -181,7 +183,17 @@ public abstract class RestEsqlTestCase extends ESRestTestCase {
         public RequestObjectBuilder build() throws IOException {
             if (isBuilt == false) {
                 if (tables != null) {
-                    builder.field("tables", tables);
+                    builder.startObject("tables");
+                    for (var table : tables.entrySet()) {
+                        builder.startObject(table.getKey());
+                        for (var column : table.getValue().entrySet()) {
+                            builder.startObject(column.getKey());
+                            builder.field(column.getValue().type(), column.getValue().values());
+                            builder.endObject();
+                        }
+                        builder.endObject();
+                    }
+                    builder.endObject();
                 }
                 builder.endObject();
                 isBuilt = true;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -57,6 +57,11 @@ public class EsqlCapabilities {
      */
     private static final String LOOKUP_COMMAND = "lookup_command";
 
+    /**
+     * Support for the syntax {@code "tables": {"type": [<values>]}}.
+     */
+    private static final String TABLES_TYPES = "tables_types";
+
     /**
      * Support for requesting the "REPEAT" command.
      */

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ParseTables.java

@@ -83,17 +83,11 @@ public class ParseTables {
                         return columns;
                     }
                     case FIELD_NAME -> {
-                        String[] fname = p.currentName().split(":");
-                        if (fname.length != 2) {
-                            throw new XContentParseException(
-                                p.getTokenLocation(),
-                                "expected columns named name:type but was [" + p.currentName() + "]"
-                            );
+                        String name = p.currentName();
+                        if (columns.containsKey(name)) {
+                            throw new XContentParseException(p.getTokenLocation(), "duplicate column name [" + name + "]");
                         }
-                        if (columns.containsKey(fname[0])) {
-                            throw new XContentParseException(p.getTokenLocation(), "duplicate column name [" + fname[0] + "]");
-                        }
-                        columns.put(fname[0], parseColumn(fname[1]));
+                        columns.put(name, parseColumn());
                     }
                     default -> throw new XContentParseException(
                         p.getTokenLocation(),
@@ -108,14 +102,26 @@ public class ParseTables {
         }
     }
 
-    private Column parseColumn(String type) throws IOException {
-        return switch (type) {
+    private Column parseColumn() throws IOException {
+        if (p.nextToken() != XContentParser.Token.START_OBJECT) {
+            throw new XContentParseException(p.getTokenLocation(), "expected " + XContentParser.Token.START_OBJECT);
+        }
+        if (p.nextToken() != XContentParser.Token.FIELD_NAME) {
+            throw new XContentParseException(p.getTokenLocation(), "expected " + XContentParser.Token.FIELD_NAME);
+        }
+        String type = p.currentName();
+        Column result = switch (type) {
             case "integer" -> parseIntColumn();
             case "keyword" -> parseKeywordColumn();
             case "long" -> parseLongColumn();
             case "double" -> parseDoubleColumn();
             default -> throw new XContentParseException(p.getTokenLocation(), "unsupported type [" + type + "]");
         };
+        if (p.nextToken() != XContentParser.Token.END_OBJECT) {
+            result.close();
+            throw new XContentParseException(p.getTokenLocation(), "expected " + XContentParser.Token.END_OBJECT);
+        }
+        return result;
     }
 
     private Column parseKeywordColumn() throws IOException {

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java

@@ -192,7 +192,7 @@ public class EsqlQueryRequestTests extends ESTestCase {
         String json = """
             {
                 "query": "ROW x = 1",
-                "tables": {"a": {"c:keyword": ["a", "b", null, 1, 2.0, ["c", "d"], false]}}
+                "tables": {"a": {"c": {"keyword": ["a", "b", null, 1, 2.0, ["c", "d"], false]}}}
             }
             """;
         EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
@@ -223,7 +223,7 @@ public class EsqlQueryRequestTests extends ESTestCase {
         String json = """
             {
                 "query": "ROW x = 1",
-                "tables": {"a": {"c:integer": [1, 2, "3", null, [5, 6]]}}
+                "tables": {"a": {"c": {"integer": [1, 2, "3", null, [5, 6]]}}}
             }
             """;
 
@@ -251,7 +251,7 @@ public class EsqlQueryRequestTests extends ESTestCase {
         String json = """
             {
                 "query": "ROW x = 1",
-                "tables": {"a": {"c:long": [1, 2, "3", null, [5, 6]]}}
+                "tables": {"a": {"c": {"long": [1, 2, "3", null, [5, 6]]}}}
             }
             """;
 
@@ -279,7 +279,7 @@ public class EsqlQueryRequestTests extends ESTestCase {
         String json = """
             {
                 "query": "ROW x = 1",
-                "tables": {"a": {"c:double": [1.1, 2, "3.1415", null, [5.1, "-6"]]}}
+                "tables": {"a": {"c": {"double": [1.1, 2, "3.1415", null, [5.1, "-6"]]}}}
             }
             """;
 
@@ -309,16 +309,16 @@ public class EsqlQueryRequestTests extends ESTestCase {
                 "query": "ROW x = 1",
                 "tables": {
                     "t1": {
-                        "a:long": [1],
-                        "b:long": [1],
-                        "c:keyword": [1],
-                        "d:long": [1]
+                        "a": {"long": [1]},
+                        "b": {"long": [1]},
+                        "c": {"keyword": [1]},
+                        "d": {"long": [1]}
                     },
                     "t2": {
-                        "a:long": [1],
-                        "b:integer": [1],
-                        "c:long": [1],
-                        "d:long": [1]
+                        "a": {"long": [1]},
+                        "b": {"integer": [1]},
+                        "c": {"long": [1]},
+                        "d": {"long": [1]}
                     }
                 }
             }

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml

@@ -38,7 +38,7 @@ basic:
         - method: POST
           path: /_query
           parameters: []
-          capabilities: [lookup_command]
+          capabilities: [lookup_command, tables_types]
       reason: "uses LOOKUP"
 
   - do:
@@ -48,8 +48,8 @@ basic:
           columnar: true
           tables:
             colors:
-              "color:keyword":  ["red", "green", "blue"]
-              "rgb:integer": [16711680,   65280,    255]
+              color: { keyword:  ["red", "green", "blue"] }
+              rgb: { integer: [16711680,   65280,    255] }
 
   - match: {columns.0.name: "color"}
   - match: {columns.0.type: "keyword"}
@@ -66,7 +66,7 @@ read multivalue keyword:
         - method: POST
           path: /_query
           parameters: []
-          capabilities: [lookup_command]
+          capabilities: [lookup_command, tables_types]
       reason: "uses LOOKUP"
 
   - do:
@@ -76,11 +76,12 @@ read multivalue keyword:
           columnar: true
           tables:
             color_associations:
-              "color:keyword":  ["red", "green", "blue"]
-              "association:keyword":
-                - ["love", "passion", "blood", "happiness"]
-                - ["nature", "healing", "health", "youth"]
-                - ["serenity", "wisdom", "ocean", "sky"]
+              color: {keyword: ["red", "green", "blue"] }
+              association:
+                keyword:
+                  - ["love", "passion", "blood", "happiness"]
+                  - ["nature", "healing", "health", "youth"]
+                  - ["serenity", "wisdom", "ocean", "sky"]
 
   - match: {columns.0.name: "color"}
   - match: {columns.0.type: "keyword"}
@@ -97,7 +98,7 @@ keyword matches text:
         - method: POST
           path: /_query
           parameters: []
-          capabilities: [lookup_command]
+          capabilities: [lookup_command, tables_types]
       reason: "uses LOOKUP"
 
   - do:
@@ -125,8 +126,8 @@ keyword matches text:
           columnar: true
           tables:
             colors:
-              "color:keyword":  ["red", "green", "blue"]
-              "rgb:integer": [16711680,   65280,    255]
+              color: { keyword:   ["red", "green", "blue"] }
+              rgb: { integer:  [16711680,   65280,    255] }
 
   - match: {columns.0.name: "color"}
   - match: {columns.0.type: "text"}
@@ -135,28 +136,6 @@ keyword matches text:
   - match: {values.0: ["red"]}
   - match: {values.1: [16711680]}
 
----
-duplicate column names in table:
-  - requires:
-      test_runner_features: [capabilities]
-      capabilities:
-        - method: POST
-          path: /_query
-          parameters: []
-          capabilities: [lookup_command]
-      reason: "uses LOOKUP"
-
-  - do:
-      catch: /duplicate column name \[color\]/
-      esql.query:
-        body:
-          query: 'FROM test | LOOKUP colors ON color | SORT time | KEEP color, rgb | LIMIT 2'
-          columnar: true
-          tables:
-            colors:
-              "color:keyword":    ["red", "green", "blue"]
-              "color:integer": [16711680,   65280,    255]
-
 ---
 duplicate keys:
   - requires:
@@ -165,7 +144,7 @@ duplicate keys:
         - method: POST
           path: /_query
           parameters: []
-          capabilities: [lookup_command]
+          capabilities: [lookup_command, tables_types]
       reason: "uses LOOKUP"
 
   - do:
@@ -177,8 +156,8 @@ duplicate keys:
           columnar: true
           tables:
             colors:
-              "color:keyword":  ["red",   "red", "blue"]
-              "rgb:integer": [16711680,   65280,    255]
+              color: {keyword:  ["red",   "red", "blue"] }
+              rgb: {integer: [16711680,   65280,    255] }
 
 ---
 multivalued keys:
@@ -188,7 +167,7 @@ multivalued keys:
         - method: POST
           path: /_query
           parameters: []
-          capabilities: [lookup_command]
+          capabilities: [lookup_command, tables_types]
       reason: "uses LOOKUP"
 
   - do:
@@ -200,8 +179,8 @@ multivalued keys:
           columnar: true
           tables:
             colors:
-              "color:keyword":  [["red", "blue"],   "white", "blue"]
-              "rgb:integer":           [16711680,     65280,    255]
+              color: { keyword: [["red", "blue"],   "white", "blue"] }
+              rgb: { integer:          [16711680,     65280,    255] }
 
 ---
 index named lookup still works:
@@ -230,7 +209,7 @@ on function:
         - method: POST
           path: /_query
           parameters: []
-          capabilities: [lookup_command]
+          capabilities: [lookup_command, tables_types]
       reason: "uses LOOKUP"
 
   - do:
@@ -241,5 +220,5 @@ on function:
           columnar: true
           tables:
             colors:
-              "color:keyword":  ["red", "green", "blue"]
-              "rgb:integer": [16711680,   65280,    255]
+              color:  { keyword: ["red", "green", "blue"] }
+              rgb: { integer: [16711680,   65280,    255] }