SQL: ignore UNSUPPORTED fields for JDBC and ODBC modes in 'SYS COLUMNS' (#39518)

* SYS COLUMNS will skip UNSUPPORTED field types in ODBC and JDBC, as well.
NESTED and OBJECT types were already skipped in ODBC mode, now they are
skipped in JDBC mode, as well.

docs/reference/sql/limitations.asciidoc

@@ -7,8 +7,8 @@
 === Nested fields in `SYS COLUMNS` and `DESCRIBE TABLE`
 
 {es} has a special type of relationship fields called `nested` fields. In {es-sql} they can be used by referencing their inner
-sub-fields. Even though `SYS COLUMNS` and `DESCRIBE TABLE` will still display them as having the type `NESTED`, they cannot
-be used in a query. One can only reference its sub-fields in the form:
+sub-fields. Even though `SYS COLUMNS` in non-driver mode (in the CLI and in REST calls) and `DESCRIBE TABLE` will still display
+them as having the type `NESTED`, they cannot be used in a query. One can only reference its sub-fields in the form:
 
 [source, sql]
 --------------------------------------------------

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java

@@ -96,8 +96,8 @@ public class SysColumns extends Command {
 
     @Override
     public void execute(SqlSession session, ActionListener<SchemaRowSet> listener) {
-        boolean isOdbcClient = session.configuration().mode() == Mode.ODBC;
-        List<Attribute> output = output(isOdbcClient);
+        Mode mode = session.configuration().mode();
+        List<Attribute> output = output(mode == Mode.ODBC);
         String cluster = session.indexResolver().clusterName();
 
         // bail-out early if the catalog is present but differs
@@ -114,7 +114,7 @@ public class SysColumns extends Command {
         session.indexResolver().resolveAsSeparateMappings(idx, regex, ActionListener.wrap(esIndices -> {
             List<List<?>> rows = new ArrayList<>();
             for (EsIndex esIndex : esIndices) {
-                fillInRows(cluster, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, isOdbcClient);
+                fillInRows(cluster, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, mode);
             }
 
             listener.onResponse(Rows.of(output, rows));
@@ -122,8 +122,9 @@ public class SysColumns extends Command {
     }
 
     static void fillInRows(String clusterName, String indexName, Map<String, EsField> mapping, String prefix, List<List<?>> rows,
-            Pattern columnMatcher, boolean isOdbcClient) {
+            Pattern columnMatcher, Mode mode) {
         int pos = 0;
+        boolean isOdbcClient = mode == Mode.ODBC;
         for (Map.Entry<String, EsField> entry : mapping.entrySet()) {
             pos++; // JDBC is 1-based so we start with 1 here
 
@@ -132,9 +133,8 @@ public class SysColumns extends Command {
             EsField field = entry.getValue();
             DataType type = field.getDataType();
             
-            // skip the nested and object types only for ODBC
-            // https://github.com/elastic/elasticsearch/issues/35376
-            if (type.isPrimitive() || !isOdbcClient) {
+            // skip the nested, object and unsupported types for JDBC and ODBC
+            if (type.isPrimitive() || false == Mode.isDriver(mode)) {
                 if (columnMatcher == null || columnMatcher.matcher(name).matches()) {
                     rows.add(asList(clusterName,
                             // schema is not supported
@@ -174,7 +174,7 @@ public class SysColumns extends Command {
                 }
             }
             if (field.getProperties() != null) {
-                fillInRows(clusterName, indexName, field.getProperties(), name, rows, columnMatcher, isOdbcClient);
+                fillInRows(clusterName, indexName, field.getProperties(), name, rows, columnMatcher, mode);
             }
         }
     }

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java

@@ -226,7 +226,7 @@ public enum DataType {
     }
 
     public boolean isPrimitive() {
-        return this != OBJECT && this != NESTED;
+        return this != OBJECT && this != NESTED && this != UNSUPPORTED;
     }
 
     public boolean isDateBased() {

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.plan.logical.command.sys;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.sql.proto.Mode;
 import org.elasticsearch.xpack.sql.type.TypesTests;
 
 import java.sql.Types;
@@ -16,8 +17,9 @@ public class SysColumnsTests extends ESTestCase {
 
     public void testSysColumns() {
         List<List<?>> rows = new ArrayList<>();
-        SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, false);
-        assertEquals(16, rows.size());
+        SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null,
+                randomValueOtherThanMany(Mode::isDriver, () -> randomFrom(Mode.values())));
+        assertEquals(17, rows.size());
         assertEquals(24, rows.get(0).size());
 
         List<?> row = rows.get(0);
@@ -37,6 +39,12 @@ public class SysColumnsTests extends ESTestCase {
         assertEquals(Types.VARCHAR, sqlType(row));
         assertEquals(null, radix(row));
         assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        
+        row = rows.get(3);
+        assertEquals("keyword", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
 
         row = rows.get(4);
         assertEquals("date", name(row));
@@ -45,23 +53,84 @@ public class SysColumnsTests extends ESTestCase {
         assertEquals(24, precision(row));
         assertEquals(8, bufferLength(row));
 
+        row = rows.get(5);
+        assertEquals("unsupported", name(row));
+        assertEquals(Types.OTHER, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(0, bufferLength(row));
+        
+        row = rows.get(6);
+        assertEquals("some", name(row));
+        assertEquals(Types.STRUCT, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(-1, bufferLength(row));
+        
         row = rows.get(7);
         assertEquals("some.dotted", name(row));
         assertEquals(Types.STRUCT, sqlType(row));
         assertEquals(null, radix(row));
         assertEquals(-1, bufferLength(row));
+        
+        row = rows.get(8);
+        assertEquals("some.dotted.field", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        
+        row = rows.get(9);
+        assertEquals("some.string", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        
+        row = rows.get(10);
+        assertEquals("some.string.normalized", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        
+        row = rows.get(11);
+        assertEquals("some.string.typical", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        
+        row = rows.get(12);
+        assertEquals("some.ambiguous", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        
+        row = rows.get(13);
+        assertEquals("some.ambiguous.one", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        
+        row = rows.get(14);
+        assertEquals("some.ambiguous.two", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
 
         row = rows.get(15);
         assertEquals("some.ambiguous.normalized", name(row));
         assertEquals(Types.VARCHAR, sqlType(row));
         assertEquals(null, radix(row));
         assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        
+        row = rows.get(16);
+        assertEquals("foo_type", name(row));
+        assertEquals(Types.OTHER, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(0, bufferLength(row));
     }
 
     public void testSysColumnsInOdbcMode() {
         List<List<?>> rows = new ArrayList<>();
-        SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, true);
-        assertEquals(14, rows.size());
+        SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, 
+                Mode.ODBC);
+        assertEquals(13, rows.size());
         assertEquals(24, rows.get(0).size());
 
         List<?> row = rows.get(0);
@@ -112,18 +181,17 @@ public class SysColumnsTests extends ESTestCase {
         assertEquals(Short.class, sqlDataTypeSub(row).getClass());
 
         row = rows.get(5);
-        assertEquals("unsupported", name(row));
-        assertEquals((short) Types.OTHER, sqlType(row));
+        assertEquals("some.dotted.field", name(row));
+        assertEquals((short) Types.VARCHAR, sqlType(row));
         assertEquals(null, radix(row));
-        assertEquals(0, precision(row));
-        assertEquals(0, bufferLength(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
         assertNull(decimalPrecision(row));
         assertEquals(Short.class, nullable(row).getClass());
         assertEquals(Short.class, sqlDataType(row).getClass());
         assertEquals(Short.class, sqlDataTypeSub(row).getClass());
 
         row = rows.get(6);
-        assertEquals("some.dotted.field", name(row));
+        assertEquals("some.string", name(row));
         assertEquals((short) Types.VARCHAR, sqlType(row));
         assertEquals(null, radix(row));
         assertEquals(Integer.MAX_VALUE, bufferLength(row));
@@ -133,7 +201,7 @@ public class SysColumnsTests extends ESTestCase {
         assertEquals(Short.class, sqlDataTypeSub(row).getClass());
 
         row = rows.get(7);
-        assertEquals("some.string", name(row));
+        assertEquals("some.string.normalized", name(row));
         assertEquals((short) Types.VARCHAR, sqlType(row));
         assertEquals(null, radix(row));
         assertEquals(Integer.MAX_VALUE, bufferLength(row));
@@ -143,7 +211,7 @@ public class SysColumnsTests extends ESTestCase {
         assertEquals(Short.class, sqlDataTypeSub(row).getClass());
 
         row = rows.get(8);
-        assertEquals("some.string.normalized", name(row));
+        assertEquals("some.string.typical", name(row));
         assertEquals((short) Types.VARCHAR, sqlType(row));
         assertEquals(null, radix(row));
         assertEquals(Integer.MAX_VALUE, bufferLength(row));
@@ -151,9 +219,29 @@ public class SysColumnsTests extends ESTestCase {
         assertEquals(Short.class, nullable(row).getClass());
         assertEquals(Short.class, sqlDataType(row).getClass());
         assertEquals(Short.class, sqlDataTypeSub(row).getClass());
-
+        
         row = rows.get(9);
-        assertEquals("some.string.typical", name(row));
+        assertEquals("some.ambiguous", name(row));
+        assertEquals((short) Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Short.class, nullable(row).getClass());
+        assertEquals(Short.class, sqlDataType(row).getClass());
+        assertEquals(Short.class, sqlDataTypeSub(row).getClass());
+        
+        row = rows.get(10);
+        assertEquals("some.ambiguous.one", name(row));
+        assertEquals((short) Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Short.class, nullable(row).getClass());
+        assertEquals(Short.class, sqlDataType(row).getClass());
+        assertEquals(Short.class, sqlDataTypeSub(row).getClass());
+        
+        row = rows.get(11);
+        assertEquals("some.ambiguous.two", name(row));
         assertEquals((short) Types.VARCHAR, sqlType(row));
         assertEquals(null, radix(row));
         assertEquals(Integer.MAX_VALUE, bufferLength(row));
@@ -162,7 +250,7 @@ public class SysColumnsTests extends ESTestCase {
         assertEquals(Short.class, sqlDataType(row).getClass());
         assertEquals(Short.class, sqlDataTypeSub(row).getClass());
 
-        row = rows.get(13);
+        row = rows.get(12);
         assertEquals("some.ambiguous.normalized", name(row));
         assertEquals((short) Types.VARCHAR, sqlType(row));
         assertEquals(null, radix(row));
@@ -172,6 +260,141 @@ public class SysColumnsTests extends ESTestCase {
         assertEquals(Short.class, sqlDataType(row).getClass());
         assertEquals(Short.class, sqlDataTypeSub(row).getClass());
     }
+    
+    public void testSysColumnsInJdbcMode() {
+        List<List<?>> rows = new ArrayList<>();
+        SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, 
+                Mode.JDBC);
+        assertEquals(13, rows.size());
+        assertEquals(24, rows.get(0).size());
+
+        List<?> row = rows.get(0);
+        assertEquals("bool", name(row));
+        assertEquals(Types.BOOLEAN, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(1, bufferLength(row));
+
+        row = rows.get(1);
+        assertEquals("int", name(row));
+        assertEquals(Types.INTEGER, sqlType(row));
+        assertEquals(Integer.class, radix(row).getClass());
+        assertEquals(4, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+
+        row = rows.get(2);
+        assertEquals("text", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+
+        row = rows.get(3);
+        assertEquals("keyword", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+
+        row = rows.get(4);
+        assertEquals("date", name(row));
+        assertEquals(Types.TIMESTAMP, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(24, precision(row));
+        assertEquals(8, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+
+        row = rows.get(5);
+        assertEquals("some.dotted.field", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+
+        row = rows.get(6);
+        assertEquals("some.string", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+
+        row = rows.get(7);
+        assertEquals("some.string.normalized", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+
+        row = rows.get(8);
+        assertEquals("some.string.typical", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+        
+        row = rows.get(9);
+        assertEquals("some.ambiguous", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+        
+        row = rows.get(10);
+        assertEquals("some.ambiguous.one", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+        
+        row = rows.get(11);
+        assertEquals("some.ambiguous.two", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+
+        row = rows.get(12);
+        assertEquals("some.ambiguous.normalized", name(row));
+        assertEquals(Types.VARCHAR, sqlType(row));
+        assertEquals(null, radix(row));
+        assertEquals(Integer.MAX_VALUE, bufferLength(row));
+        assertNull(decimalPrecision(row));
+        assertEquals(Integer.class, nullable(row).getClass());
+        assertEquals(Integer.class, sqlDataType(row).getClass());
+        assertEquals(Integer.class, sqlDataTypeSub(row).getClass());
+    }
 
     private static Object name(List<?> list) {
         return list.get(3);

x-pack/plugin/sql/src/test/resources/mapping-multi-field-variation.json

@@ -43,6 +43,7 @@
                     }
                 }
             }
-        }
+        },
+        "foo_type" : { "type" : "foo" }
     }
 }