Support insert dynamic values by column-based (#690)

Signed-off-by: yhmo <yihua.mo@zilliz.com>

src/main/java/io/milvus/param/Constant.java

@@ -44,6 +44,7 @@ public class Constant {
     public static final String DEFAULT_INDEX_NAME = "";
     public final static String OFFSET = "offset";
     public final static String LIMIT = "limit";
+    public final static String DYNAMIC_FIELD_NAME = "$meta";
 
     // constant values for general
     public static final String TTL_SECONDS = "collection.ttl.seconds";

src/main/java/io/milvus/param/ParamUtils.java

@@ -268,13 +268,15 @@ public class ParamUtils {
         List<JSONObject> rowFields = requestParam.getRows();
 
         if (CollectionUtils.isNotEmpty(columnFields)) {
-            checkAndSetColumnData(requestParam, wrapper.getFields(), insertBuilder, columnFields);
+            checkAndSetColumnData(wrapper, insertBuilder, columnFields);
         } else {
             checkAndSetRowData(wrapper, insertBuilder, rowFields);
         }
     }
 
-    private static void checkAndSetColumnData(InsertParam requestParam, List<FieldType> fieldTypes, InsertRequest.Builder insertBuilder, List<InsertParam.Field> fields) {
+    private static void checkAndSetColumnData(DescCollResponseWrapper wrapper, InsertRequest.Builder insertBuilder, List<InsertParam.Field> fields) {
+        List<FieldType> fieldTypes = wrapper.getFields();
+
         // gen fieldData
         // make sure the field order must be consisted with collection schema
         for (FieldType fieldType : fieldTypes) {
@@ -288,7 +290,7 @@ public class ParamUtils {
                     checkFieldData(fieldType, field);
 
                     found = true;
-                    insertBuilder.addFieldsData(genFieldData(field.getName(), fieldType.getDataType(), field.getValues()));
+                    insertBuilder.addFieldsData(genFieldData(fieldType, field.getValues()));
                     break;
                 }
 
@@ -298,18 +300,40 @@ public class ParamUtils {
                 throw new ParamException(msg);
             }
         }
+
+        // deal with dynamicField
+        if (wrapper.getEnableDynamicField()) {
+            for (InsertParam.Field field : fields) {
+                if (field.getName().equals(Constant.DYNAMIC_FIELD_NAME)) {
+                    FieldType dynamicType = FieldType.newBuilder()
+                            .withName(Constant.DYNAMIC_FIELD_NAME)
+                            .withDataType(DataType.JSON)
+                            .withIsDynamic(true)
+                            .build();
+                    checkFieldData(dynamicType, field);
+                    insertBuilder.addFieldsData(genFieldData(dynamicType, field.getValues(), true));
+                    break;
+                }
+            }
+        }
     }
 
     private static void checkAndSetRowData(DescCollResponseWrapper wrapper, InsertRequest.Builder insertBuilder, List<JSONObject> rows) {
         List<FieldType> fieldTypes = wrapper.getFields();
 
         Map<String, InsertDataInfo> nameInsertInfo = new HashMap<>();
-        InsertDataInfo insertDynamicDataInfo = InsertDataInfo.builder().dataType(DataType.JSON).data(new LinkedList<>()).build();
+        InsertDataInfo insertDynamicDataInfo = InsertDataInfo.builder().fieldType(
+                FieldType.newBuilder()
+                        .withName(Constant.DYNAMIC_FIELD_NAME)
+                        .withDataType(DataType.JSON)
+                        .withIsDynamic(true)
+                        .build())
+                .data(new LinkedList<>()).build();
         for (JSONObject row : rows) {
             for (FieldType fieldType : fieldTypes) {
                 String fieldName = fieldType.getName();
                 InsertDataInfo insertDataInfo = nameInsertInfo.getOrDefault(fieldName, InsertDataInfo.builder()
-                        .fieldName(fieldName).dataType(fieldType.getDataType()).data(new LinkedList<>()).build());
+                        .fieldType(fieldType).data(new LinkedList<>()).build());
 
                 // check normalField
                 Object rowFieldData = row.get(fieldName);
@@ -345,10 +369,10 @@ public class ParamUtils {
 
         for (String fieldNameKey : nameInsertInfo.keySet()) {
             InsertDataInfo insertDataInfo = nameInsertInfo.get(fieldNameKey);
-            insertBuilder.addFieldsData(genFieldData(insertDataInfo.getFieldName(), insertDataInfo.getDataType(), insertDataInfo.getData()));
+            insertBuilder.addFieldsData(genFieldData(insertDataInfo.getFieldType(), insertDataInfo.getData()));
         }
         if (wrapper.getEnableDynamicField()) {
-            insertBuilder.addFieldsData(genFieldData(insertDynamicDataInfo.getFieldName(), insertDynamicDataInfo.getDataType(), insertDynamicDataInfo.getData(), Boolean.TRUE));
+            insertBuilder.addFieldsData(genFieldData(insertDynamicDataInfo.getFieldType(), insertDynamicDataInfo.getData(), Boolean.TRUE));
         }
     }
 
@@ -544,15 +568,17 @@ public class ParamUtils {
         add(DataType.BinaryVector);
     }};
 
-    private static FieldData genFieldData(String fieldName, DataType dataType, List<?> objects) {
-        return genFieldData(fieldName, dataType, objects, Boolean.FALSE);
+    private static FieldData genFieldData(FieldType fieldType, List<?> objects) {
+        return genFieldData(fieldType, objects, Boolean.FALSE);
     }
 
     @SuppressWarnings("unchecked")
-    private static FieldData genFieldData(String fieldName, DataType dataType, List<?> objects, boolean isDynamic) {
+    private static FieldData genFieldData(FieldType fieldType, List<?> objects, boolean isDynamic) {
         if (objects == null) {
             throw new ParamException("Cannot generate FieldData from null object");
         }
+        DataType dataType = fieldType.getDataType();
+        String fieldName = fieldType.getName();
         FieldData.Builder builder = FieldData.newBuilder();
         if (vectorDataType.contains(dataType)) {
             if (dataType == DataType.FloatVector) {
@@ -719,8 +745,7 @@ public class ParamUtils {
     @Builder
     @Getter
     public static class InsertDataInfo {
-        private final String fieldName;
-        private final DataType dataType;
+        private final FieldType fieldType;
         private final LinkedList<Object> data;
     }
 }

src/test/java/io/milvus/client/MilvusClientDockerTest.java

@@ -1244,11 +1244,7 @@ class MilvusClientDockerTest {
         indexTypes.put(IndexType.IVF_FLAT, "{\"nlist\":128}");
         indexTypes.put(IndexType.IVF_SQ8, "{\"nlist\":128}");
         indexTypes.put(IndexType.IVF_PQ, "{\"nlist\":128, \"m\":16, \"nbits\":8}");
-        indexTypes.put(IndexType.ANNOY, "{\"n_trees\":16}");
         indexTypes.put(IndexType.HNSW, "{\"M\":16,\"efConstruction\":64}");
-        indexTypes.put(IndexType.RHNSW_FLAT, "{\"M\":16,\"efConstruction\":64}");
-        indexTypes.put(IndexType.RHNSW_PQ, "{\"M\":16,\"efConstruction\":64, \"PQM\":16}");
-        indexTypes.put(IndexType.RHNSW_SQ, "{\"M\":16,\"efConstruction\":64}");
 
         List<MetricType> metricTypes = new ArrayList<>();
         metricTypes.add(MetricType.L2);
@@ -1297,8 +1293,8 @@ class MilvusClientDockerTest {
 
         // test all supported indexes
         List<MetricType> flatMetricTypes = new ArrayList<>();
-        flatMetricTypes.add(MetricType.SUBSTRUCTURE);
-        flatMetricTypes.add(MetricType.SUPERSTRUCTURE);
+        flatMetricTypes.add(MetricType.HAMMING);
+        flatMetricTypes.add(MetricType.JACCARD);
 
         for (MetricType metric : flatMetricTypes) {
             testIndex(randomCollectionName, field2Name, IndexType.BIN_FLAT, metric, "{}", Boolean.TRUE);
@@ -1308,7 +1304,6 @@ class MilvusClientDockerTest {
         List<MetricType> ivfMetricTypes = new ArrayList<>();
         ivfMetricTypes.add(MetricType.HAMMING);
         ivfMetricTypes.add(MetricType.JACCARD);
-        ivfMetricTypes.add(MetricType.TANIMOTO);
 
         for (MetricType metric : ivfMetricTypes) {
             testIndex(randomCollectionName, field2Name, IndexType.BIN_IVF_FLAT, metric, "{\"nlist\":128}", Boolean.TRUE);
@@ -1394,11 +1389,11 @@ class MilvusClientDockerTest {
 
             // JSON field
             JSONObject info = new JSONObject();
-            info.put("row-based-info", i);
+            info.put("row_based_info", i);
             row.put(field3Name, info);
 
             // extra meta is automatically stored in dynamic field
-            row.put("extra_meta", i % 3 == 0);
+            row.put("row_based_extra", i % 3 == 0);
             row.put(generator.generate(5), 100);
 
             rows.add(row);
@@ -1416,12 +1411,17 @@ class MilvusClientDockerTest {
         // insert data by column-based
         List<Long> ids = new ArrayList<>();
         List<JSONObject> infos = new ArrayList<>();
+        List<JSONObject> dynamics = new ArrayList<>();
         for (long i = 0L; i < rowCount; ++i) {
             ids.add(rowCount + i);
             JSONObject obj = new JSONObject();
-            obj.put("column-based-info", i);
+            obj.put("column_based_info", i);
             obj.put(generator.generate(5), i);
             infos.add(obj);
+
+            JSONObject dynamic = new JSONObject();
+            dynamic.put(String.format("column_based_extra_%d", i), i);
+            dynamics.add(dynamic);
         }
         List<List<Float>> vectors = generateFloatVectors(rowCount);
 
@@ -1429,6 +1429,7 @@ class MilvusClientDockerTest {
         fieldsInsert.add(new InsertParam.Field(field1Name, ids));
         fieldsInsert.add(new InsertParam.Field(field2Name, vectors));
         fieldsInsert.add(new InsertParam.Field(field3Name, infos));
+        fieldsInsert.add(new InsertParam.Field(Constant.DYNAMIC_FIELD_NAME, dynamics));
 
         InsertParam insertColumnsParam = InsertParam.newBuilder()
                 .withCollectionName(randomCollectionName)
@@ -1451,8 +1452,8 @@ class MilvusClientDockerTest {
         System.out.println("Collection row count: " + stat.getRowCount());
 
         // retrieve rows
-        String expr = "extra_meta == true";
-        List<String> outputFields = Arrays.asList(field3Name, "extra_meta");
+        String expr = "row_based_extra == true";
+        List<String> outputFields = Arrays.asList(field3Name, "row_based_extra");
         QueryParam queryParam = QueryParam.newBuilder()
                 .withCollectionName(randomCollectionName)
                 .withExpr(expr)
@@ -1464,13 +1465,11 @@ class MilvusClientDockerTest {
 
         QueryResultsWrapper queryResultsWrapper = new QueryResultsWrapper(queryR.getData());
         List<QueryResultsWrapper.RowRecord> records = queryResultsWrapper.getRowRecords();
-        System.out.println("Query results:");
+        System.out.println("Query results with expr: " + expr);
         for (QueryResultsWrapper.RowRecord record:records) {
             System.out.println(record);
-            Object extraMeta = record.get("extra_meta");
-            if (extraMeta != null) {
-                System.out.println("'extra_meta' is from dynamic field, value: " + extraMeta);
-            }
+            Object extraMeta = record.get("row_based_extra");
+            System.out.println("'row_based_extra' is from dynamic field, value: " + extraMeta);
         }
 
         // search
@@ -1496,13 +1495,33 @@ class MilvusClientDockerTest {
             System.out.println("The result of No." + i + " target vector:");
             for (SearchResultsWrapper.IDScore score:scores) {
                 System.out.println(score);
-                Object extraMeta = score.get("extra_meta");
+                Object extraMeta = score.get("row_based_extra");
                 if (extraMeta != null) {
-                    System.out.println("'extra_meta' is from dynamic field, value: " + extraMeta);
+                    System.out.println("'row_based_extra' is from dynamic field, value: " + extraMeta);
                 }
             }
         }
 
+        // retrieve dynamic values inserted by column-based
+        expr = "column_based_extra_1 == 1";
+        queryParam = QueryParam.newBuilder()
+                .withCollectionName(randomCollectionName)
+                .withExpr("column_based_extra_1 == 1")
+                .withOutFields(Collections.singletonList("*"))
+                .build();
+
+        queryR = client.query(queryParam);
+        Assertions.assertEquals(R.Status.Success.getCode(), queryR.getStatus().intValue());
+
+        queryResultsWrapper = new QueryResultsWrapper(queryR.getData());
+        records = queryResultsWrapper.getRowRecords();
+        System.out.println("Query results with expr: " + expr);
+        for (QueryResultsWrapper.RowRecord record:records) {
+            System.out.println(record);
+            long id = (long)record.get(field1Name);
+            Assertions.assertEquals((long)rowCount+1L, id);
+        }
+
         // drop collection
         R<RpcStatus> dropR = client.dropCollection(DropCollectionParam.newBuilder()
                 .withCollectionName(randomCollectionName)