Support insert dynamic values by column-based (#674)

Signed-off-by: yhmo <yihua.mo@zilliz.com>

src/main/java/io/milvus/param/ParamUtils.java

@@ -325,13 +325,15 @@ public class ParamUtils {
             List<JSONObject> rowFields = requestParam.getRows();
 
             if (CollectionUtils.isNotEmpty(columnFields)) {
-                checkAndSetColumnData(wrapper.getFields(), columnFields);
+                checkAndSetColumnData(wrapper, columnFields);
             } else {
                 checkAndSetRowData(wrapper, rowFields);
             }
         }
 
-        private void checkAndSetColumnData(List<FieldType> fieldTypes, List<InsertParam.Field> fields) {
+        private void checkAndSetColumnData(DescCollResponseWrapper wrapper, List<InsertParam.Field> fields) {
+            List<FieldType> fieldTypes = wrapper.getFields();
+
             // gen fieldData
             // make sure the field order must be consisted with collection schema
             for (FieldType fieldType : fieldTypes) {
@@ -355,6 +357,22 @@ public class ParamUtils {
                     throw new ParamException(msg);
                 }
             }
+
+            // deal with dynamicField
+            if (wrapper.getEnableDynamicField()) {
+                for (InsertParam.Field field : fields) {
+                    if (field.getName().equals(Constant.DYNAMIC_FIELD_NAME)) {
+                        FieldType dynamicType = FieldType.newBuilder()
+                                .withName(Constant.DYNAMIC_FIELD_NAME)
+                                .withDataType(DataType.JSON)
+                                .withIsDynamic(true)
+                                .build();
+                        checkFieldData(dynamicType, field);
+                        this.addFieldsData(genFieldData(dynamicType, field.getValues(), true));
+                        break;
+                    }
+                }
+            }
         }
 
         private void checkAndSetRowData(DescCollResponseWrapper wrapper, List<JSONObject> rows) {

src/test/java/io/milvus/client/MilvusClientDockerTest.java

@@ -1389,11 +1389,11 @@ class MilvusClientDockerTest {
 
             // JSON field
             JSONObject info = new JSONObject();
-            info.put("row-based-info", i);
+            info.put("row_based_info", i);
             row.put(field3Name, info);
 
             // extra meta is automatically stored in dynamic field
-            row.put("extra_meta", i % 3 == 0);
+            row.put("row_based_extra", i % 3 == 0);
             row.put(generator.generate(5), 100);
 
             rows.add(row);
@@ -1411,12 +1411,17 @@ class MilvusClientDockerTest {
         // insert data by column-based
         List<Long> ids = new ArrayList<>();
         List<JSONObject> infos = new ArrayList<>();
+        List<JSONObject> dynamics = new ArrayList<>();
         for (long i = 0L; i < rowCount; ++i) {
             ids.add(rowCount + i);
             JSONObject obj = new JSONObject();
-            obj.put("column-based-info", i);
+            obj.put("column_based_info", i);
             obj.put(generator.generate(5), i);
             infos.add(obj);
+
+            JSONObject dynamic = new JSONObject();
+            dynamic.put(String.format("column_based_extra_%d", i), i);
+            dynamics.add(dynamic);
         }
         List<List<Float>> vectors = generateFloatVectors(rowCount);
 
@@ -1424,6 +1429,7 @@ class MilvusClientDockerTest {
         fieldsInsert.add(new InsertParam.Field(field1Name, ids));
         fieldsInsert.add(new InsertParam.Field(field2Name, vectors));
         fieldsInsert.add(new InsertParam.Field(field3Name, infos));
+        fieldsInsert.add(new InsertParam.Field(Constant.DYNAMIC_FIELD_NAME, dynamics));
 
         InsertParam insertColumnsParam = InsertParam.newBuilder()
                 .withCollectionName(randomCollectionName)
@@ -1446,8 +1452,8 @@ class MilvusClientDockerTest {
         System.out.println("Collection row count: " + stat.getRowCount());
 
         // retrieve rows
-        String expr = "extra_meta == true";
-        List<String> outputFields = Arrays.asList(field3Name, "extra_meta");
+        String expr = "row_based_extra == true";
+        List<String> outputFields = Arrays.asList(field3Name, "row_based_extra");
         QueryParam queryParam = QueryParam.newBuilder()
                 .withCollectionName(randomCollectionName)
                 .withExpr(expr)
@@ -1459,11 +1465,11 @@ class MilvusClientDockerTest {
 
         QueryResultsWrapper queryResultsWrapper = new QueryResultsWrapper(queryR.getData());
         List<QueryResultsWrapper.RowRecord> records = queryResultsWrapper.getRowRecords();
-        System.out.println("Query results:");
+        System.out.println("Query results with expr: " + expr);
         for (QueryResultsWrapper.RowRecord record:records) {
             System.out.println(record);
-            Object extraMeta = record.get("extra_meta");
-            System.out.println("'extra_meta' is from dynamic field, value: " + extraMeta);
+            Object extraMeta = record.get("row_based_extra");
+            System.out.println("'row_based_extra' is from dynamic field, value: " + extraMeta);
         }
 
         // search the No.11 and No.15
@@ -1491,15 +1497,35 @@ class MilvusClientDockerTest {
             System.out.println("The result of No." + i + " target vector:");
             for (SearchResultsWrapper.IDScore score:scores) {
                 System.out.println(score);
-                Object extraMeta = score.get("extra_meta");
+                Object extraMeta = score.get("row_based_extra");
                 if (extraMeta != null) {
-                    System.out.println("'extra_meta' is from dynamic field, value: " + extraMeta);
+                    System.out.println("'row_based_extra' is from dynamic field, value: " + extraMeta);
                 }
             }
         }
         Assertions.assertEquals(results.getIDScore(0).get(0).getLongID(), 11L);
         Assertions.assertEquals(results.getIDScore(1).get(0).getLongID(), 15L);
 
+        // retrieve dynamic values inserted by column-based
+        expr = "column_based_extra_1 == 1";
+        queryParam = QueryParam.newBuilder()
+                .withCollectionName(randomCollectionName)
+                .withExpr("column_based_extra_1 == 1")
+                .withOutFields(Collections.singletonList("*"))
+                .build();
+
+        queryR = client.query(queryParam);
+        Assertions.assertEquals(R.Status.Success.getCode(), queryR.getStatus().intValue());
+
+        queryResultsWrapper = new QueryResultsWrapper(queryR.getData());
+        records = queryResultsWrapper.getRowRecords();
+        System.out.println("Query results with expr: " + expr);
+        for (QueryResultsWrapper.RowRecord record:records) {
+            System.out.println(record);
+            long id = (long)record.get(field1Name);
+            Assertions.assertEquals((long)rowCount+1L, id);
+        }
+
         // drop collection
         R<RpcStatus> dropR = client.dropCollection(DropCollectionParam.newBuilder()
                 .withCollectionName(randomCollectionName)