SQL: introduce the columnar option for REST requests (#39287)

* Add "columnar" option for REST requests (but be lenient for non-"plain"
modes) for json, yaml, smile and cbor formats.
* Updated documentation

docs/reference/sql/endpoints/rest.asciidoc

@@ -133,7 +133,7 @@ Which returns:
 [float]
 === Paginating through a large response
 
-Using the example above, onu can continue to the next page by sending back the `cursor` field. In
+Using the example above, one can continue to the next page by sending back the `cursor` field. In
 case of text format the cursor is returned as `Cursor` http header.
 
 [source,js]
@@ -235,6 +235,81 @@ Douglas Adams  |The Hitchhiker's Guide to the Galaxy|180            |1979-10-12T
 // TESTRESPONSE[s/\|/\\|/ s/\+/\\+/]
 // TESTRESPONSE[_cat]
 
+[[sql-rest-columnar]]
+[float]
+=== Columnar results
+
+The most well known way of displaying the results of an SQL query result in general is the one where each
+individual record/document represents one line/row. For certain formats, {es-sql} can return the results
+in a columnar fashion: one row represents all the values of a certain column from the current page of results.
+
+The following formats can be returned in columnar orientation: `json`, `yaml`, `cbor` and `smile`.
+
+[source,js]
+--------------------------------------------------
+POST /_sql?format=json
+{
+    "query": "SELECT * FROM library ORDER BY page_count DESC",
+    "fetch_size": 5,
+    "columnar": true
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:library]
+
+Which returns:
+
+[source,js]
+--------------------------------------------------
+{
+    "columns": [
+        {"name": "author", "type": "text"},
+        {"name": "name", "type": "text"},
+        {"name": "page_count", "type": "short"},
+        {"name": "release_date", "type": "datetime"}
+    ],
+    "values": [
+        ["Peter F. Hamilton", "Vernor Vinge", "Frank Herbert", "Alastair Reynolds", "James S.A. Corey"],
+        ["Pandora's Star", "A Fire Upon the Deep", "Dune", "Revelation Space", "Leviathan Wakes"],
+        [768, 613, 604, 585, 561],
+        ["2004-03-02T00:00:00.000Z", "1992-06-01T00:00:00.000Z", "1965-06-01T00:00:00.000Z", "2000-03-15T00:00:00.000Z", "2011-06-02T00:00:00.000Z"]
+    ],
+    "cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8="
+}
+--------------------------------------------------
+// TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl\+v\/\/\/w8=/$body.cursor/]
+
+Any subsequent calls using a `cursor` still have to contain the `columnar` parameter to preserve the orientation,
+meaning the initial query will not _remember_ the columnar option.
+
+[source,js]
+--------------------------------------------------
+POST /_sql?format=json
+{
+    "cursor": "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8=",
+    "columnar": true
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+// TEST[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl\+v\/\/\/w8=/$body.cursor/]
+
+Which looks like:
+
+[source,js]
+--------------------------------------------------
+{
+    "values": [
+        ["Dan Simmons", "Iain M. Banks", "Neal Stephenson", "Frank Herbert", "Frank Herbert"],
+        ["Hyperion", "Consider Phlebas", "Snow Crash", "God Emperor of Dune", "Children of Dune"],
+        [482, 471, 470, 454, 408],
+        ["1989-05-26T00:00:00.000Z", "1987-04-23T00:00:00.000Z", "1992-06-01T00:00:00.000Z", "1981-05-28T00:00:00.000Z", "1976-04-21T00:00:00.000Z"]
+    ],
+    "cursor": "46ToAwFzQERYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFBQUVXWjBaNlFXbzNOV0pVY21Wa1NUZDJhV2t3V2xwblp3PT3/////DwQBZgZhdXRob3IBBHRleHQAAAFmBG5hbWUBBHRleHQAAAFmCnBhZ2VfY291bnQBBGxvbmcBAAFmDHJlbGVhc2VfZGF0ZQEIZGF0ZXRpbWUBAAEP"
+}
+--------------------------------------------------
+// TESTRESPONSE[s/46ToAwFzQERYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFBQUVXWjBaNlFXbzNOV0pVY21Wa1NUZDJhV2t3V2xwblp3PT3\/\/\/\/\/DwQBZgZhdXRob3IBBHRleHQAAAFmBG5hbWUBBHRleHQAAAFmCnBhZ2VfY291bnQBBGxvbmcBAAFmDHJlbGVhc2VfZGF0ZQEIZGF0ZXRpbWUBAAEP/$body.cursor/]
+
 [[sql-rest-fields]]
 [float]
 === Supported REST parameters
@@ -277,7 +352,11 @@ s|Description
 |Time-zone in ISO 8601 used for executing the query on the server.
 More information available https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html[here].
 
+|columnar
+|false
+|Return the results in a columnar fashion, rather than row-based fashion. Valid for `json`, `yaml`, `cbor` and `smile`.
+
 |===
 
-Do note that most parameters (outside the timeout ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <<sql-pagination, pagination>> chapter.
+Do note that most parameters (outside the timeout and `columnar` ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <<sql-pagination, pagination>> chapter.
 That's because the query has already been executed and the calls are simply about returning the found results - thus the parameters are simply ignored.

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java

@@ -53,7 +53,7 @@ class JdbcHttpClient {
                 SqlQueryRequest sqlRequest = new SqlQueryRequest(sql, params, null, Protocol.TIME_ZONE,
                 fetch,
                 TimeValue.timeValueMillis(meta.timeoutInMs()), TimeValue.timeValueMillis(meta.queryTimeoutInMs()),
-                new RequestInfo(Mode.JDBC));
+                false, new RequestInfo(Mode.JDBC));
         SqlQueryResponse response = httpClient.query(sqlRequest);
         return new DefaultCursor(this, response.cursor(), toJdbcColumnInfo(response.columns()), response.rows(), meta);
     }

x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/UserFunctionIT.java

@@ -105,7 +105,6 @@ public class UserFunctionIT extends ESRestTestCase {
         assertResponse(expected, actual);
     }
     
-    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/35980")
     public void testSingleRandomUserWithWhereEvaluatingFalse() throws IOException {
         index("{\"test\":\"doc1\"}",
               "{\"test\":\"doc2\"}",

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java

@@ -121,7 +121,8 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase {
     @SuppressWarnings({ "unchecked" })
     private void assertQuery(String sql, String columnName, String columnType, Object columnValue, int displaySize, Mode mode)
             throws IOException {
-        Map<String, Object> response = runSql(mode.toString(), sql);
+        boolean columnar = randomBoolean();
+        Map<String, Object> response = runSql(mode.toString(), sql, columnar);
         List<Object> columns = (ArrayList<Object>) response.get("columns");
         assertEquals(1, columns.size());
 
@@ -135,7 +136,7 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase {
             assertEquals(2, column.size());
         }
         
-        List<Object> rows = (ArrayList<Object>) response.get("rows");
+        List<Object> rows = (ArrayList<Object>) response.get(columnar == true ? "values" : "rows");
         assertEquals(1, rows.size());
         List<Object> row = (ArrayList<Object>) rows.get(0);
         assertEquals(1, row.size());
@@ -149,7 +150,7 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase {
         }
     }
     
-    private Map<String, Object> runSql(String mode, String sql) throws IOException {
+    private Map<String, Object> runSql(String mode, String sql, boolean columnar) throws IOException {
         Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
         String requestContent = "{\"query\":\"" + sql + "\"" + mode(mode) + "}";
         String format = randomFrom(XContentType.values()).name().toLowerCase(Locale.ROOT);
@@ -177,6 +178,11 @@ public abstract class SqlProtocolTestCase extends ESRestTestCase {
             options.addHeader("Accept", randomFrom("*/*", "application/" + format));
             request.setOptions(options);
         }
+        if ((false == columnar && randomBoolean()) || columnar) {
+            // randomly set the "columnar" parameter, either "true" (non-default) or explicit "false" (the default anyway)
+            requestContent = new StringBuilder(requestContent)
+                    .insert(requestContent.length() - 1, ",\"columnar\":" + columnar).toString();
+        }
         
         // send the query either as body or as request parameter
         if (randomBoolean()) {

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.qa.rest;
 
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
+
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
@@ -51,6 +52,7 @@ import static org.hamcrest.Matchers.containsString;
 public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTestCase {
     
     public static String SQL_QUERY_REST_ENDPOINT = org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT;
+    private static String SQL_TRANSLATE_REST_ENDPOINT = org.elasticsearch.xpack.sql.proto.Protocol.SQL_TRANSLATE_REST_ENDPOINT;
     /**
      * Builds that map that is returned in the header for each column.
      */
@@ -70,9 +72,15 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
 
         Map<String, Object> expected = new HashMap<>();
         String mode = randomMode();
+        boolean columnar = randomBoolean();
+        
         expected.put("columns", singletonList(columnInfo(mode, "test", "text", JDBCType.VARCHAR, 0)));
-        expected.put("rows", Arrays.asList(singletonList("test"), singletonList("test")));
-        assertResponse(expected, runSql(mode, "SELECT * FROM test"));
+        if (columnar) {
+            expected.put("values", singletonList(Arrays.asList("test", "test")));
+        } else {
+            expected.put("rows", Arrays.asList(singletonList("test"), singletonList("test")));
+        }
+        assertResponse(expected, runSql(mode, "SELECT * FROM test", columnar));
     }
 
     public void testNextPage() throws IOException {
@@ -86,14 +94,15 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
         }
         request.setJsonEntity(bulk.toString());
         client().performRequest(request);
-
+        
+        boolean columnar = randomBoolean();
         String sqlRequest =
                   "{\"query\":\""
                 + "   SELECT text, number, SQRT(number) AS s, SCORE()"
                 + "     FROM test"
                 + " ORDER BY number, SCORE()\", "
                 + "\"mode\":\"" + mode + "\", "
-            + "\"fetch_size\":2}";
+            + "\"fetch_size\":2" + columnarParameter(columnar) + "}";
 
         String cursor = null;
         for (int i = 0; i < 20; i += 2) {
@@ -101,7 +110,8 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
             if (i == 0) {
                 response = runSql(new StringEntity(sqlRequest, ContentType.APPLICATION_JSON), "");
             } else {
-                response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + "}",
+                columnar = randomBoolean();
+                response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"" + mode(mode) + columnarParameter(columnar) + "}",
                         ContentType.APPLICATION_JSON), StringUtils.EMPTY);
             }
 
@@ -113,32 +123,52 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
                         columnInfo(mode, "s", "double", JDBCType.DOUBLE, 25),
                         columnInfo(mode, "SCORE()", "float", JDBCType.REAL, 15)));
             }
-            expected.put("rows", Arrays.asList(
-                    Arrays.asList("text" + i, i, Math.sqrt(i), 1.0),
-                    Arrays.asList("text" + (i + 1), i + 1, Math.sqrt(i + 1), 1.0)));
+            
+            if (columnar) {
+                expected.put("values", Arrays.asList(
+                        Arrays.asList("text" + i, "text" + (i + 1)),
+                        Arrays.asList(i, i + 1),
+                        Arrays.asList(Math.sqrt(i), Math.sqrt(i + 1)),
+                        Arrays.asList(1.0, 1.0)));
+            } else {
+                expected.put("rows", Arrays.asList(
+                        Arrays.asList("text" + i, i, Math.sqrt(i), 1.0),
+                        Arrays.asList("text" + (i + 1), i + 1, Math.sqrt(i + 1), 1.0)));
+            }
             cursor = (String) response.remove("cursor");
             assertResponse(expected, response);
             assertNotNull(cursor);
         }
         Map<String, Object> expected = new HashMap<>();
-        expected.put("rows", emptyList());
-        assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + "}",
+        columnar = randomBoolean();
+        if (columnar) {
+            expected.put("values", emptyList());
+        } else {
+            expected.put("rows", emptyList());
+        }
+        assertResponse(expected, runSql(new StringEntity("{ \"cursor\":\"" + cursor + "\"" + mode(mode) + columnarParameter(columnar) + "}",
                 ContentType.APPLICATION_JSON), StringUtils.EMPTY));
     }
 
     @AwaitsFix(bugUrl = "Unclear status, https://github.com/elastic/x-pack-elasticsearch/issues/2074")
     public void testTimeZone() throws IOException {
         String mode = randomMode();
+        boolean columnar = randomBoolean();
         index("{\"test\":\"2017-07-27 00:00:00\"}",
             "{\"test\":\"2017-07-27 01:00:00\"}");
 
         Map<String, Object> expected = new HashMap<>();
         expected.put("columns", singletonMap("test", singletonMap("type", "text")));
-        expected.put("rows", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test")));
+        if (columnar) {
+            expected.put("values", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test")));
+        } else {
+            // TODO: what exactly is this test suppossed to do. We need to check the 2074 issue above.
+            expected.put("rows", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test")));
+        }
         expected.put("size", 2);
 
         // Default TimeZone is UTC
-        assertResponse(expected, runSql(mode, "SELECT DAY_OF_YEAR(test), COUNT(*) FROM test"));
+        assertResponse(expected, runSql(mode, "SELECT DAY_OF_YEAR(test), COUNT(*) FROM test", columnar));
     }
 
     public void testScoreWithFieldNamedScore() throws IOException {
@@ -152,15 +182,19 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
         client().performRequest(request);
 
         Map<String, Object> expected = new HashMap<>();
+        boolean columnar = randomBoolean();
         expected.put("columns", Arrays.asList(
             columnInfo(mode, "name", "text", JDBCType.VARCHAR, 0),
             columnInfo(mode, "score", "long", JDBCType.BIGINT, 20),
             columnInfo(mode, "SCORE()", "float", JDBCType.REAL, 15)));
-        expected.put("rows", singletonList(Arrays.asList(
-            "test", 10, 1.0)));
-
-        assertResponse(expected, runSql(mode, "SELECT *, SCORE() FROM test ORDER BY SCORE()"));
-        assertResponse(expected, runSql(mode, "SELECT name, \\\"score\\\", SCORE() FROM test ORDER BY SCORE()"));
+        if (columnar) {
+            expected.put("values", Arrays.asList(singletonList("test"), singletonList(10), singletonList(1.0)));
+        } else {
+            expected.put("rows", singletonList(Arrays.asList("test", 10, 1.0)));
+        }
+        
+        assertResponse(expected, runSql(mode, "SELECT *, SCORE() FROM test ORDER BY SCORE()", columnar));
+        assertResponse(expected, runSql(mode, "SELECT name, \\\"score\\\", SCORE() FROM test ORDER BY SCORE()", columnar));
     }
 
     public void testSelectWithJoinFails() throws Exception {
@@ -195,8 +229,8 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
 
     public void testSelectWhereExistsFails() throws Exception {
         index("{\"foo\":1}", "{\"foo\":2}");
-        expectBadRequest(() -> runSql(randomMode(), "SELECT foo FROM test WHERE EXISTS (SELECT * FROM test t WHERE t.foo = test.foo)"),
-            containsString("line 1:28: EXISTS is not yet supported"));
+        expectBadRequest(() -> runSql(randomMode(), "SELECT foo FROM test WHERE EXISTS (SELECT * FROM test t WHERE t.foo = test.foo)",
+                randomBoolean()), containsString("line 1:28: EXISTS is not yet supported"));
     }
 
 
@@ -280,6 +314,34 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
         expectBadRequest(() -> runSql(randomMode(), "SELECT SIN(SCORE()) FROM test"),
             containsString("line 1:12: [SCORE()] cannot be an argument to a function"));
     }
+    
+    public void testUseColumnarForUnsupportedFormats() throws Exception {
+        String format = randomFrom("txt", "csv", "tsv");
+        index("{\"foo\":1}");
+        
+        Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
+        request.addParameter("error_trace", "true");
+        request.addParameter("pretty", "true");
+        request.addParameter("format", format);
+        request.setEntity(new StringEntity("{\"columnar\":true,\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}",
+                ContentType.APPLICATION_JSON));
+        expectBadRequest(() -> {
+                client().performRequest(request);
+                return Collections.emptyMap();
+            }, containsString("Invalid use of [columnar] argument: cannot be used in combination with txt, csv or tsv formats"));
+    }
+    
+    public void testUseColumnarForTranslateRequest() throws IOException {
+        index("{\"test\":\"test\"}", "{\"test\":\"test\"}");
+        
+        Request request = new Request("POST", SQL_TRANSLATE_REST_ENDPOINT);
+        request.setEntity(new StringEntity("{\"columnar\":true,\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}",
+                ContentType.APPLICATION_JSON));
+        expectBadRequest(() -> {
+                client().performRequest(request);
+                return Collections.emptyMap();
+            }, containsString("unknown field [columnar], parser not found"));
+    }
 
     protected void expectBadRequest(CheckedSupplier<Map<String, Object>, Exception> code, Matcher<String> errorMessageMatcher) {
         try {
@@ -303,11 +365,25 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
     }
 
     private Map<String, Object> runSql(String mode, String sql) throws IOException {
-        return runSql(mode, sql, StringUtils.EMPTY);
+        return runSql(mode, sql, StringUtils.EMPTY, randomBoolean());
+    }
+    
+    private Map<String, Object> runSql(String mode, String sql, boolean columnar) throws IOException {
+        return runSql(mode, sql, StringUtils.EMPTY, columnar);
     }
 
-    private Map<String, Object> runSql(String mode, String sql, String suffix) throws IOException {
-        return runSql(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + "}", ContentType.APPLICATION_JSON), suffix);
+    private Map<String, Object> runSql(String mode, String sql, String suffix, boolean columnar) throws IOException {
+        // put an explicit "columnar": false parameter or omit it altogether, it should make no difference
+        return runSql(new StringEntity("{\"query\":\"" + sql + "\"" + mode(mode) + columnarParameter(columnar) + "}",
+                ContentType.APPLICATION_JSON), suffix);
+    }
+    
+    private String columnarParameter(boolean columnar) {
+        if (columnar == false && randomBoolean()) {
+            return "";
+        } else {
+            return ",\"columnar\":" + columnar;
+        }
     }
 
     protected Map<String, Object> runSql(HttpEntity sql, String suffix) throws IOException {
@@ -334,8 +410,9 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
 
     public void testBasicTranslateQuery() throws IOException {
         index("{\"test\":\"test\"}", "{\"test\":\"test\"}");
-
-        Map<String, Object> response = runSql(randomMode(), "SELECT * FROM test", "/translate/");
+        
+        Map<String, Object> response = runSql(new StringEntity("{\"query\":\"SELECT * FROM test\"" + mode(randomMode()) + "}",
+                ContentType.APPLICATION_JSON), "/translate/");
         assertEquals(1000, response.get("size"));
         @SuppressWarnings("unchecked")
         Map<String, Object> source = (Map<String, Object>) response.get("_source");
@@ -359,6 +436,7 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
 
     public void testBasicQueryWithParameters() throws IOException {
         String mode = randomMode();
+        boolean columnar = randomBoolean();
         index("{\"test\":\"foo\"}",
                 "{\"test\":\"bar\"}");
 
@@ -367,10 +445,14 @@ public abstract class RestSqlTestCase extends ESRestTestCase implements ErrorsTe
                 columnInfo(mode, "test", "text", JDBCType.VARCHAR, 0),
                 columnInfo(mode, "param", "integer", JDBCType.INTEGER, 11)
         ));
-        expected.put("rows", singletonList(Arrays.asList("foo", 10)));
+        if (columnar) {
+            expected.put("values", Arrays.asList(singletonList("foo"), singletonList(10)));
+        } else {
+            expected.put("rows", Arrays.asList(Arrays.asList("foo", 10)));
+        }
         assertResponse(expected, runSql(new StringEntity("{\"query\":\"SELECT test, ? param FROM test WHERE test = ?\", " +
                 "\"params\":[{\"type\": \"integer\", \"value\": 10}, {\"type\": \"keyword\", \"value\": \"foo\"}]"
-                + mode(mode) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY));
+                + mode(mode) + columnarParameter(columnar) + "}", ContentType.APPLICATION_JSON), StringUtils.EMPTY));
     }
 
     public void testBasicTranslateQueryWithFilter() throws IOException {

x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.action;
 
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -29,21 +30,30 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  */
 public class SqlQueryRequest extends AbstractSqlQueryRequest {
     private static final ObjectParser<SqlQueryRequest, Void> PARSER = objectParser(SqlQueryRequest::new);
+    static final ParseField COLUMNAR = new ParseField("columnar");
 
     static {
         PARSER.declareString(SqlQueryRequest::cursor, CURSOR);
+        PARSER.declareBoolean(SqlQueryRequest::columnar, COLUMNAR);
     }
 
     private String cursor = "";
+    /*
+     * Using the Boolean object here so that SqlTranslateRequest to set this to null (since it doesn't need a "columnar" parameter).
+     * See {@code SqlTranslateRequest.toXContent}
+     */
+    private Boolean columnar = Boolean.FALSE;
 
     public SqlQueryRequest() {
         super();
     }
 
     public SqlQueryRequest(String query, List<SqlTypedParamValue> params, QueryBuilder filter, ZoneId zoneId,
-                           int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, String cursor, RequestInfo requestInfo) {
+                           int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, Boolean columnar,
+                           String cursor, RequestInfo requestInfo) {
         super(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, requestInfo);
         this.cursor = cursor;
+        this.columnar = columnar;
     }
 
     @Override
@@ -74,26 +84,44 @@ public class SqlQueryRequest extends AbstractSqlQueryRequest {
         this.cursor = cursor;
         return this;
     }
+    
+    /**
+     * Should format the values in a columnar fashion or not (default false).
+     * Depending on the format used (csv, tsv, txt, json etc) this setting will be taken into
+     * consideration or not, depending on whether it even makes sense for that specific format or not.
+     */
+    public Boolean columnar() {
+        return columnar;
+    }
+
+    public SqlQueryRequest columnar(boolean columnar) {
+        this.columnar = columnar;
+        return this;
+    }
 
     public SqlQueryRequest(StreamInput in) throws IOException {
         super(in);
         cursor = in.readString();
+        columnar = in.readOptionalBoolean();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(cursor);
+        out.writeOptionalBoolean(columnar);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), cursor);
+        return Objects.hash(super.hashCode(), cursor, columnar);
     }
 
     @Override
     public boolean equals(Object obj) {
-        return super.equals(obj) && Objects.equals(cursor, ((SqlQueryRequest) obj).cursor);
+        return super.equals(obj) 
+                && Objects.equals(cursor, ((SqlQueryRequest) obj).cursor)
+                && Objects.equals(columnar, ((SqlQueryRequest) obj).columnar);
     }
 
     @Override
@@ -105,7 +133,7 @@ public class SqlQueryRequest extends AbstractSqlQueryRequest {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         // This is needed just to test round-trip compatibility with proto.SqlQueryRequest
         return new org.elasticsearch.xpack.sql.proto.SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(),
-            pageTimeout(), filter(), cursor(), requestInfo()).toXContent(builder, params);
+            pageTimeout(), filter(), columnar(), cursor(), requestInfo()).toXContent(builder, params);
     }
 
     public static SqlQueryRequest fromXContent(XContentParser parser) {

x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestBuilder.java

@@ -25,14 +25,14 @@ public class SqlQueryRequestBuilder extends ActionRequestBuilder<SqlQueryRequest
 
     public SqlQueryRequestBuilder(ElasticsearchClient client, SqlQueryAction action) {
         this(client, action, "", Collections.emptyList(), null, Protocol.TIME_ZONE, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT,
-            Protocol.PAGE_TIMEOUT, "", new RequestInfo(Mode.PLAIN));
+            Protocol.PAGE_TIMEOUT, false, "", new RequestInfo(Mode.PLAIN));
     }
 
     public SqlQueryRequestBuilder(ElasticsearchClient client, SqlQueryAction action, String query, List<SqlTypedParamValue> params,
             QueryBuilder filter, ZoneId zoneId, int fetchSize, TimeValue requestTimeout,
-                                  TimeValue pageTimeout, String nextPageInfo, RequestInfo requestInfo) {
-        super(client, action, new SqlQueryRequest(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, nextPageInfo,
-                requestInfo));
+            TimeValue pageTimeout, boolean columnar, String nextPageInfo, RequestInfo requestInfo) {
+        super(client, action, new SqlQueryRequest(query, params, filter, zoneId, fetchSize, requestTimeout, pageTimeout, columnar,
+                nextPageInfo, requestInfo));
     }
 
     public SqlQueryRequestBuilder query(String query) {
@@ -74,6 +74,11 @@ public class SqlQueryRequestBuilder extends ActionRequestBuilder<SqlQueryRequest
         request.pageTimeout(timeout);
         return this;
     }
+    
+    public SqlQueryRequestBuilder columnar(boolean columnar) {
+        request.columnar(columnar);
+        return this;
+    }
 
     public SqlQueryRequestBuilder fetchSize(int fetchSize) {
         request.fetchSize(fetchSize);

x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java

@@ -34,6 +34,7 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject
     // TODO: Simplify cursor handling
     private String cursor;
     private Mode mode;
+    private boolean columnar;
     private List<ColumnInfo> columns;
     // TODO investigate reusing Page here - it probably is much more efficient
     private List<List<Object>> rows;
@@ -42,9 +43,10 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject
     public SqlQueryResponse() {
     }
 
-    public SqlQueryResponse(String cursor, Mode mode, @Nullable List<ColumnInfo> columns, List<List<Object>> rows) {
+    public SqlQueryResponse(String cursor, Mode mode, boolean columnar, @Nullable List<ColumnInfo> columns, List<List<Object>> rows) {
         this.cursor = cursor;
         this.mode = mode;
+        this.columnar = columnar;
         this.columns = columns;
         this.rows = rows;
     }
@@ -64,6 +66,10 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject
     public List<ColumnInfo> columns() {
         return columns;
     }
+    
+    public boolean columnar() {
+        return columnar;
+    }
 
     public List<List<Object>> rows() {
         return rows;
@@ -150,15 +156,35 @@ public class SqlQueryResponse extends ActionResponse implements ToXContentObject
                 }
                 builder.endArray();
             }
-            builder.startArray("rows");
-            for (List<Object> row : rows()) {
-                builder.startArray();
-                for (Object value : row) {
-                    value(builder, mode, value);
+            
+            if (columnar) {
+                // columns can be specified (for the first REST request for example), or not (on a paginated/cursor based request)
+                // if the columns are missing, we take the first rows' size as the number of columns
+                long columnsCount = columns != null ? columns.size() : 0;
+                if (size() > 0) {
+                    columnsCount = rows().get(0).size();
+                }
+
+                builder.startArray("values");
+                for (int index = 0; index < columnsCount; index++) {
+                    builder.startArray();
+                    for (List<Object> row : rows()) {
+                        value(builder, mode, row.get(index));
+                    }
+                    builder.endArray();
+                }
+                builder.endArray();
+            } else {
+                builder.startArray("rows");
+                for (List<Object> row : rows()) {
+                    builder.startArray();
+                    for (Object value : row) {
+                        value(builder, mode, value);
+                    }
+                    builder.endArray();
                 }
                 builder.endArray();
             }
-            builder.endArray();
 
             if (cursor.equals("") == false) {
                 builder.field(CURSOR.getPreferredName(), cursor);

x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequest.java

@@ -65,7 +65,7 @@ public class SqlTranslateRequest extends AbstractSqlQueryRequest {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         // This is needed just to test parsing of SqlTranslateRequest, so we can reuse SqlQuerySerialization
         return new SqlQueryRequest(query(), params(), zoneId(), fetchSize(), requestTimeout(),
-            pageTimeout(), filter(), null, requestInfo()).toXContent(builder, params);
+            pageTimeout(), filter(), null, null, requestInfo()).toXContent(builder, params);
 
     }
 }

x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java

@@ -54,8 +54,8 @@ public class SqlQueryRequestTests extends AbstractSerializingTestCase<SqlQueryRe
     @Override
     protected SqlQueryRequest createTestInstance() {
         return new SqlQueryRequest(randomAlphaOfLength(10), randomParameters(), SqlTestUtils.randomFilterOrNull(random()),
-                randomZone(), between(1, Integer.MAX_VALUE),
-                randomTV(), randomTV(), randomAlphaOfLength(10), requestInfo
+                randomZone(), between(1, Integer.MAX_VALUE), randomTV(),
+                randomTV(), randomBoolean(), randomAlphaOfLength(10), requestInfo
         );
     }
     
@@ -109,11 +109,12 @@ public class SqlQueryRequestTests extends AbstractSerializingTestCase<SqlQueryRe
                 request -> request.requestTimeout(randomValueOtherThan(request.requestTimeout(), this::randomTV)),
                 request -> request.filter(randomValueOtherThan(request.filter(),
                         () -> request.filter() == null ? randomFilter(random()) : randomFilterOrNull(random()))),
+                request -> request.columnar(randomValueOtherThan(request.columnar(), () -> randomBoolean())),
                 request -> request.cursor(randomValueOtherThan(request.cursor(), SqlQueryResponseTests::randomStringCursor))
         );
         SqlQueryRequest newRequest = new SqlQueryRequest(instance.query(), instance.params(), instance.filter(),
-                instance.zoneId(), instance.fetchSize(), instance.requestTimeout(), instance.pageTimeout(), instance.cursor(),
-                instance.requestInfo());
+                instance.zoneId(), instance.fetchSize(), instance.requestTimeout(), instance.pageTimeout(), instance.columnar(),
+                instance.cursor(), instance.requestInfo());
         mutator.accept(newRequest);
         return newRequest;
     }

x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java

@@ -36,10 +36,10 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
 
     @Override
     protected SqlQueryResponse createTestInstance() {
-        return createRandomInstance(randomStringCursor(), randomFrom(Mode.values()));
+        return createRandomInstance(randomStringCursor(), randomFrom(Mode.values()), randomBoolean());
     }
 
-    public static SqlQueryResponse createRandomInstance(String cursor, Mode mode) {
+    public static SqlQueryResponse createRandomInstance(String cursor, Mode mode, boolean columnar) {
         int columnCount = between(1, 10);
 
         List<ColumnInfo> columns = null;
@@ -55,6 +55,12 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
             rows = Collections.emptyList();
         } else {
             int rowCount = between(1, 10);
+            if (columnar && columns != null) {
+                int temp = rowCount;
+                rowCount = columnCount;
+                columnCount = temp;
+            }
+            
             rows = new ArrayList<>(rowCount);
             for (int r = 0; r < rowCount; r++) {
                 List<Object> row = new ArrayList<>(rowCount);
@@ -65,12 +71,11 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
                             ESTestCase::randomDouble,
                             () -> null));
                     row.add(value.get());
-
                 }
                 rows.add(row);
             }
         }
-        return new SqlQueryResponse(cursor, mode, columns, rows);
+        return new SqlQueryResponse(cursor, mode, false, columns, rows);
     }
 
     @Override
@@ -100,7 +105,13 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
             assertNull(rootMap.get("columns"));
         }
 
-        List<?> rows = ((List<?>) rootMap.get("rows"));
+        List<?> rows;
+        if (testInstance.columnar()) {
+            rows = ((List<?>) rootMap.get("values"));
+        } else {
+            rows = ((List<?>) rootMap.get("rows"));
+        }
+        assertNotNull(rows);
         assertThat(rows, hasSize(testInstance.rows().size()));
         for (int i = 0; i < rows.size(); i++) {
             List<?> row = (List<?>) rows.get(i);
@@ -116,6 +127,6 @@ public class SqlQueryResponseTests extends AbstractStreamableXContentTestCase<Sq
     protected SqlQueryResponse doParseInstance(XContentParser parser) {
         org.elasticsearch.xpack.sql.proto.SqlQueryResponse response =
             org.elasticsearch.xpack.sql.proto.SqlQueryResponse.fromXContent(parser);
-        return new SqlQueryResponse(response.cursor(), Mode.JDBC, response.columns(), response.rows());
+        return new SqlQueryResponse(response.cursor(), Mode.JDBC, false, response.columns(), response.rows());
     }
 }

x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestTests.java

@@ -36,7 +36,7 @@ public class SqlTranslateRequestTests extends AbstractSerializingTestCase<SqlTra
 
     @Override
     protected SqlTranslateRequest createTestInstance() {
-        return new SqlTranslateRequest(randomAlphaOfLength(10),  Collections.emptyList(), randomFilterOrNull(random()),
+        return new SqlTranslateRequest(randomAlphaOfLength(10), Collections.emptyList(), randomFilterOrNull(random()),
                 randomZone(), between(1, Integer.MAX_VALUE), randomTV(), randomTV(), new RequestInfo(testMode));
     }
 

x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/HttpClient.java

@@ -66,7 +66,7 @@ public class HttpClient {
         // method called only from CLI
         SqlQueryRequest sqlRequest = new SqlQueryRequest(query, Collections.emptyList(), null, ZoneId.of("Z"),
             fetchSize, TimeValue.timeValueMillis(cfg.queryTimeout()), TimeValue.timeValueMillis(cfg.pageTimeout()),
-            new RequestInfo(Mode.CLI));
+            false, new RequestInfo(Mode.CLI));
         return query(sqlRequest);
     }
 

x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryRequest.java

@@ -30,11 +30,13 @@ public class SqlQueryRequest extends AbstractSqlRequest {
     private final TimeValue pageTimeout;
     @Nullable
     private final ToXContent filter;
+    private final Boolean columnar;
     private final List<SqlTypedParamValue> params;
 
 
     public SqlQueryRequest(String query, List<SqlTypedParamValue> params, ZoneId zoneId, int fetchSize,
-                           TimeValue requestTimeout, TimeValue pageTimeout, ToXContent filter, String cursor, RequestInfo requestInfo) {
+                           TimeValue requestTimeout, TimeValue pageTimeout, ToXContent filter, Boolean columnar,
+                           String cursor, RequestInfo requestInfo) {
         super(requestInfo);
         this.query = query;
         this.params = params;
@@ -43,17 +45,18 @@ public class SqlQueryRequest extends AbstractSqlRequest {
         this.requestTimeout = requestTimeout;
         this.pageTimeout = pageTimeout;
         this.filter = filter;
+        this.columnar = columnar;
         this.cursor = cursor;
     }
 
     public SqlQueryRequest(String query, List<SqlTypedParamValue> params, ToXContent filter, ZoneId zoneId,
-                           int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo) {
-        this(query, params, zoneId, fetchSize, requestTimeout, pageTimeout, filter, null, requestInfo);
+                           int fetchSize, TimeValue requestTimeout, TimeValue pageTimeout, boolean columnar, RequestInfo requestInfo) {
+        this(query, params, zoneId, fetchSize, requestTimeout, pageTimeout, filter, columnar, null, requestInfo);
     }
 
     public SqlQueryRequest(String cursor, TimeValue requestTimeout, TimeValue pageTimeout, RequestInfo requestInfo) {
         this("", Collections.emptyList(), Protocol.TIME_ZONE, Protocol.FETCH_SIZE, requestTimeout, pageTimeout,
-             null, cursor, requestInfo);
+             null, false, cursor, requestInfo);
     }
 
     /**
@@ -113,6 +116,14 @@ public class SqlQueryRequest extends AbstractSqlRequest {
     public ToXContent filter() {
         return filter;
     }
+    
+    /**
+     * Optional setting for returning the result values in a columnar fashion (as opposed to rows of values).
+     * Each column will have all its values in a list. Defaults to false.
+     */
+    public Boolean columnar() {
+        return columnar;
+    }
 
     @Override
     public boolean equals(Object o) {
@@ -133,12 +144,13 @@ public class SqlQueryRequest extends AbstractSqlRequest {
             Objects.equals(requestTimeout, that.requestTimeout) &&
             Objects.equals(pageTimeout, that.pageTimeout) &&
             Objects.equals(filter, that.filter) &&
+            Objects.equals(columnar,  that.columnar) &&
             Objects.equals(cursor, that.cursor);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), query, zoneId, fetchSize, requestTimeout, pageTimeout, filter, cursor);
+        return Objects.hash(super.hashCode(), query, zoneId, fetchSize, requestTimeout, pageTimeout, filter, columnar, cursor);
     }
 
     @Override
@@ -173,6 +185,9 @@ public class SqlQueryRequest extends AbstractSqlRequest {
             builder.field("filter");
             filter.toXContent(builder, params);
         }
+        if (columnar != null) {
+            builder.field("columnar", columnar);
+        }
         if (cursor != null) {
             builder.field("cursor", cursor);
         }

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java

@@ -104,6 +104,13 @@ public class RestSqlQueryAction extends BaseRestHandler {
 
         TextFormat textFormat = TextFormat.fromMediaTypeOrFormat(accept);
 
+        // if we reached this point, the format to be used can be one of TXT, CSV or TSV
+        // which won't work in a columnar fashion
+        if (sqlRequest.columnar()) {
+            throw new IllegalArgumentException("Invalid use of [columnar] argument: cannot be used in combination with "
+                    + "txt, csv or tsv formats");
+        }
+        
         long startNanos = System.nanoTime();
         return channel -> client.execute(SqlQueryAction.INSTANCE, sqlRequest, new RestResponseListener<SqlQueryResponse>(channel) {
             @Override

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java

@@ -79,7 +79,8 @@ public class TransportSqlQueryAction extends HandledTransportAction<SqlQueryRequ
                     wrap(rowSet -> listener.onResponse(createResponse(request, rowSet)), listener::onFailure));
         } else {
             planExecutor.nextPage(cfg, Cursors.decodeFromString(request.cursor()),
-                    wrap(rowSet -> listener.onResponse(createResponse(request.mode(), rowSet, null)), listener::onFailure));
+                    wrap(rowSet -> listener.onResponse(createResponse(request.mode(), request.columnar(), rowSet, null)),
+                            listener::onFailure));
         }
     }
 
@@ -93,10 +94,10 @@ public class TransportSqlQueryAction extends HandledTransportAction<SqlQueryRequ
             }
         }
         columns = unmodifiableList(columns);
-        return createResponse(request.mode(), rowSet, columns);
+        return createResponse(request.mode(), request.columnar(), rowSet, columns);
     }
 
-    static SqlQueryResponse createResponse(Mode mode, RowSet rowSet, List<ColumnInfo> columns) {
+    static SqlQueryResponse createResponse(Mode mode, boolean columnar, RowSet rowSet, List<ColumnInfo> columns) {
         List<List<Object>> rows = new ArrayList<>();
         rowSet.forEachRow(rowView -> {
             List<Object> row = new ArrayList<>(rowView.columnCount());
@@ -107,6 +108,7 @@ public class TransportSqlQueryAction extends HandledTransportAction<SqlQueryRequ
         return new SqlQueryResponse(
                 Cursors.encodeToString(Version.CURRENT, rowSet.nextPageCursor()),
                 mode,
+                columnar,
                 columns,
                 rows);
     }

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/BasicFormatterTests.java

@@ -17,7 +17,7 @@ import static org.hamcrest.Matchers.arrayWithSize;
 
 public class BasicFormatterTests extends ESTestCase {
     private final FormatOption format = randomFrom(FormatOption.values());
-    private final SqlQueryResponse firstResponse = new SqlQueryResponse("", format == CLI ? Mode.CLI : Mode.PLAIN,
+    private final SqlQueryResponse firstResponse = new SqlQueryResponse("", format == CLI ? Mode.CLI : Mode.PLAIN, false,
             Arrays.asList(
                     new ColumnInfo("", "foo", "string", 0),
                     new ColumnInfo("", "bar", "long", 15),

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CursorTests.java

@@ -72,7 +72,7 @@ public class CursorTests extends ESTestCase {
                 columns.add(new ColumnInfo(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), randomInt(25)));
             }
         }
-        return new SqlQueryResponse("", randomFrom(Mode.values()), columns, Collections.emptyList());
+        return new SqlQueryResponse("", randomFrom(Mode.values()), false, columns, Collections.emptyList());
     }
 
     @SuppressWarnings("unchecked")

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/TextFormatTests.java

@@ -118,7 +118,7 @@ public class TextFormatTests extends ESTestCase {
     }
 
     private static SqlQueryResponse emptyData() {
-        return new SqlQueryResponse(null, Mode.JDBC, singletonList(new ColumnInfo("index", "name", "keyword")), emptyList());
+        return new SqlQueryResponse(null, Mode.JDBC, false, singletonList(new ColumnInfo("index", "name", "keyword")), emptyList());
     }
 
     private static SqlQueryResponse regularData() {
@@ -132,7 +132,7 @@ public class TextFormatTests extends ESTestCase {
         values.add(asList("Along The River Bank", 11 * 60 + 48));
         values.add(asList("Mind Train", 4 * 60 + 40));
 
-        return new SqlQueryResponse(null, Mode.JDBC, headers, values);
+        return new SqlQueryResponse(null, Mode.JDBC, false, headers, values);
     }
 
     private static SqlQueryResponse escapedData() {
@@ -146,7 +146,7 @@ public class TextFormatTests extends ESTestCase {
         values.add(asList("normal", "\"quo\"ted\",\n"));
         values.add(asList("commas", "a,b,c,\n,d,e,\t\n"));
 
-        return new SqlQueryResponse(null, Mode.JDBC, headers, values);
+        return new SqlQueryResponse(null, Mode.JDBC, false, headers, values);
     }
 
     private static RestRequest req() {