Add testcases about add field (#1494)

Signed-off-by: yongpengli-z <yongpeng.li@zilliz.com>

tests/milvustestv2/pom.xml

@@ -70,7 +70,7 @@
         <dependency>
             <groupId>io.milvus</groupId>
             <artifactId>milvus-sdk-java</artifactId>
-            <version>2.5.5</version>
+            <version>2.6.0</version>
         </dependency>
         <dependency>
             <groupId>com.google.protobuf</groupId>

tests/milvustestv2/src/main/java/com/zilliz/milvustestv2/common/CommonData.java

@@ -1,61 +1,65 @@
 package com.zilliz.milvustestv2.common;
 
+import io.milvus.v2.common.DataType;
+
 /**
  * @Author yongpeng.li
  * @Date 2024/1/31 18:03
  */
 public class CommonData {
-    public static long numberEntities=5000;
-    public static long batchSize=1000;
-    public static int dim=128;
-    public static String defaultFloatVectorCollection="FloatVectorCollection";
-    public static String defaultBinaryVectorCollection="BinaryVectorCollection";
-    public static String defaultFloat16VectorCollection="Float16VectorCollection";
-    public static String defaultBFloat16VectorCollection="BFloat16VectorCollection";
-    public static String defaultSparseFloatVectorCollection="SparseFloatVectorCollection";
-    public static String partitionNameA="partitionNameA";
-    public static String partitionNameB="partitionNameB";
-    public static String partitionNameC="partitionNameC";
-    public static String defaultPartitionName="_default";
-    public static String fieldInt64="fieldInt64";
-    public static String fieldInt32="fieldInt32";
-    public static String fieldInt16="fieldInt16";
-    public static String fieldInt8="fieldInt8";
-    public static String fieldDouble="fieldDouble";
-    public static String fieldArray="fieldArray";
-    public static String fieldBool="fieldBool";
-    public static String fieldVarchar="fieldVarchar";
-    public static String fieldFloat="fieldFloat";
-    public static String fieldJson="fieldJson";
-
-    public static String fieldDynamic="fieldDynamic";
-    public static String fieldDynamicNotExist="fieldDynamicNotExist";
-    public static String fieldFloatVector="fieldFloatVector";
-    public static String fieldBinaryVector="fieldBinaryVector";
-    public static String fieldFloat16Vector="fieldFloat16Vector";
-    public static String fieldBF16Vector="fieldBF16Vector";
-    public static String fieldSparseVector="fieldSparseVector";
-
-
-    public static String partitionName="partitionName";
+    public static long numberEntities = 5000;
+    public static long batchSize = 1000;
+    public static int dim = 128;
+    public static int maxCapacity = 100;
+    public static String defaultFloatVectorCollection = "FloatVectorCollection";
+    public static String defaultBinaryVectorCollection = "BinaryVectorCollection";
+    public static String defaultFloat16VectorCollection = "Float16VectorCollection";
+    public static String defaultBFloat16VectorCollection = "BFloat16VectorCollection";
+    public static String defaultSparseFloatVectorCollection = "SparseFloatVectorCollection";
+    public static String partitionNameA = "partitionNameA";
+    public static String partitionNameB = "partitionNameB";
+    public static String partitionNameC = "partitionNameC";
+    public static String defaultPartitionName = "_default";
+    public static String fieldInt64 = "fieldInt64";
+    public static String fieldInt32 = "fieldInt32";
+    public static String fieldInt16 = "fieldInt16";
+    public static String fieldInt8 = "fieldInt8";
+    public static String fieldDouble = "fieldDouble";
+    public static String fieldArray = "fieldArray";
+    public static String fieldBool = "fieldBool";
+    public static String fieldVarchar = "fieldVarchar";
+    public static String fieldFloat = "fieldFloat";
+    public static String fieldJson = "fieldJson";
+    public static String dynamicField="dynamicField";
+    public static String fieldDynamic = "fieldDynamic";
+    public static String fieldDynamicNotExist = "fieldDynamicNotExist";
+    public static String fieldFloatVector = "fieldFloatVector";
+    public static String fieldBinaryVector = "fieldBinaryVector";
+    public static String fieldFloat16Vector = "fieldFloat16Vector";
+    public static String fieldBF16Vector = "fieldBF16Vector";
+    public static String fieldSparseVector = "fieldSparseVector";
+    public static int addMaxLength = 99;
+
+
+    public static String partitionName = "partitionName";
     // 快速创建时候的默认向量filed
-    public static String simpleVector="vector";
-    public static String simplePk="id";
-    public static String alias="ColAlias";
+    public static String simpleVector = "vector";
+    public static String simplePk = "id";
+    public static String alias = "ColAlias";
 
-    public static int topK=10;
-    public static int nq=1;
+    public static int topK = 10;
+    public static int nq = 1;
 
-    public static String rootUser="root";
+    public static String rootUser = "root";
 
-    public static String userName="user1";
-    public static String password="password1";
+    public static String userName = "user1";
+    public static String password = "password1";
 
-    public static String roleName="role02";
+    public static String roleName = "role02";
 
-    public static String databaseName="database00";
-    public static String databaseName2="database02";
-    public static String databaseName1="database01";
+    public static String databaseName = "database00";
+    public static String databaseName2 = "database02";
+    public static String databaseName1 = "database01";
 
     public static short defaultValueShort = 1;
     public static int defaultValueInt = 1;
@@ -65,12 +69,47 @@ public class CommonData {
     public static String defaultValueString = "1.0";
 
     public static int groupSize = 3;
-    public static String resourceGroup="resourceGroup01";
-    public static String defaultResourceGroup="__default_resource_group";
-    public static String resourceGroup2="resourceGroup02";
-
-
-
+    public static String resourceGroup = "resourceGroup01";
+    public static String defaultResourceGroup = "__default_resource_group";
+    public static String resourceGroup2 = "resourceGroup02";
+
+
+    public static String providerFieldNameByDatatype(DataType dataType) {
+        switch (dataType) {
+            case VarChar:
+                return fieldVarchar;
+            case Int64:
+                return fieldInt64;
+            case Int8:
+                return fieldInt8;
+            case Int16:
+                return fieldInt16;
+            case Int32:
+                return fieldInt32;
+            case Float:
+                return fieldFloat;
+            case Double:
+                return fieldDouble;
+            case FloatVector:
+                return fieldFloatVector;
+            case BinaryVector:
+                return fieldBinaryVector;
+            case Float16Vector:
+                return fieldFloat16Vector;
+            case BFloat16Vector:
+                return fieldBF16Vector;
+            case SparseFloatVector:
+                return fieldSparseVector;
+            case Bool:
+                return fieldBool;
+            case Array:
+                return fieldArray;
+            case JSON:
+                return fieldJson;
+            default:
+                return fieldDynamic;
+        }
+    }
 
 
 }

tests/milvustestv2/src/main/java/com/zilliz/milvustestv2/common/CommonFunction.java

@@ -1296,7 +1296,10 @@ public class CommonFunction {
         if (ifLoad) {
             milvusClientV2.loadCollection(LoadCollectionReq.builder().collectionName(collectionName).build());
         }
-        insertIntoCollectionByBatch(collectionName, numberEntities, CommonData.dim, vectorType);
+//        insertIntoCollectionByBatch(collectionName, numberEntities, CommonData.dim, vectorType);
+        List<JsonObject> jsonObjects = genCommonData(collectionName, numberEntities);
+        InsertResp insert = milvusClientV2.insert(InsertReq.builder().collectionName(collectionName).data(jsonObjects).build());
+
 
     }
 
@@ -1480,6 +1483,11 @@ public class CommonFunction {
                 }
                 row = JsonObjectUtil.jsonMerge(row, jsonObject);
             }
+            // 判断是否有动态列
+            if (describeCollectionResp.getCollectionSchema().isEnableDynamicField()) {
+                JsonObject jsonObject = generalJsonObjectByDataType(CommonData.dynamicField, DataType.JSON, 0, i);
+                row = JsonObjectUtil.jsonMerge(row, jsonObject);
+            }
             jsonList.add(row);
         }
         return jsonList;

tests/milvustestv2/src/test/java/com/zilliz/milvustestv2/collection/AddFieldTest.java

@@ -0,0 +1,218 @@
+package com.zilliz.milvustestv2.collection;
+
+import com.google.common.collect.Lists;
+import com.zilliz.milvustestv2.common.BaseTest;
+import com.zilliz.milvustestv2.common.CommonData;
+import com.zilliz.milvustestv2.common.CommonFunction;
+import io.milvus.v2.common.ConsistencyLevel;
+import io.milvus.v2.common.DataType;
+import io.milvus.v2.service.collection.request.*;
+import io.milvus.v2.service.collection.response.DescribeCollectionResp;
+import io.milvus.v2.service.vector.request.SearchReq;
+import io.milvus.v2.service.vector.request.data.BaseVector;
+import io.milvus.v2.service.vector.response.SearchResp;
+import lombok.extern.slf4j.Slf4j;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.util.List;
+
+@Slf4j
+public class AddFieldTest extends BaseTest {
+    String collectionName;
+    String collectionNameWithLoaded;
+    String collectionWithDynamicField;
+
+    @BeforeClass
+    public void initTestData() {
+        collectionName = CommonFunction.createNewCollection(CommonData.dim, null, DataType.FloatVector);
+        collectionWithDynamicField = CommonFunction.createNewCollectionWithDynamic(CommonData.dim, null, DataType.FloatVector);
+        CommonFunction.createIndexAndInsertAndLoad(collectionWithDynamicField, DataType.FloatVector, true, CommonData.numberEntities);
+        collectionNameWithLoaded = CommonFunction.createNewCollection(CommonData.dim, null, DataType.FloatVector);
+        CommonFunction.createIndexAndInsertAndLoad(collectionNameWithLoaded, DataType.FloatVector, true, CommonData.numberEntities);
+    }
+
+    @DataProvider(name = "providerDataType")
+    public Object[][] addFieldData() {
+        return new Object[][]{
+                {DataType.Bool},
+                {DataType.Int8},
+                {DataType.Int16},
+                {DataType.Int32},
+                {DataType.Int64},
+                {DataType.Float},
+                {DataType.Double},
+                {DataType.FloatVector},
+                {DataType.BinaryVector},
+                {DataType.VarChar},
+                {DataType.SparseFloatVector},
+                {DataType.Float16Vector},
+                {DataType.Int8Vector},
+                {DataType.Array},
+                {DataType.BFloat16Vector}
+        };
+    }
+
+
+    @AfterClass
+    public void cleanData() {
+        milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(collectionName).build());
+        milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(collectionWithDynamicField).build());
+        milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(collectionNameWithLoaded).build());
+    }
+
+    @Test(description = "add field without load", groups = {"Smoke"}, dataProvider = "providerDataType")
+    public void addFieldBeforeLoad(DataType dataType) {
+        AddCollectionFieldReq build = AddCollectionFieldReq.builder().collectionName(collectionName)
+                .dataType(dataType)
+                .fieldName("add_" + CommonData.providerFieldNameByDatatype(dataType))
+                .isNullable(true)
+                .build();
+        boolean vector = dataType.equals(DataType.FloatVector) || dataType.equals(DataType.BinaryVector) || dataType.equals(DataType.SparseFloatVector) || dataType.equals(DataType.Float16Vector) || dataType.equals(DataType.Int8Vector) || dataType.equals(DataType.BFloat16Vector);
+        if (vector) {
+            build.setDimension(CommonData.dim);
+        }
+        if (dataType.equals(DataType.Array)) {
+            build.setElementType(DataType.Int32);
+            build.setMaxCapacity(CommonData.maxCapacity);
+        }
+        try {
+            milvusClientV2.addCollectionField(build);
+            DescribeCollectionResp describeCollectionResp = milvusClientV2.describeCollection(DescribeCollectionReq.builder()
+                    .collectionName(collectionName)
+                    .build());
+            CreateCollectionReq.CollectionSchema collectionSchema = describeCollectionResp.getCollectionSchema();
+            CreateCollectionReq.FieldSchema field = collectionSchema.getField("add_" + CommonData.providerFieldNameByDatatype(dataType));
+            Assert.assertEquals(field.getDataType(), dataType);
+        } catch (Exception e) {
+            Assert.assertTrue(vector);
+            Assert.assertTrue(e.getMessage().contains("not support to add vector field"));
+        }
+    }
+
+    @Test(description = "dynamic field collection add field ", groups = {"Smoke"}, dataProvider = "providerDataType")
+    public void dynamicFieldCollectionAddField(DataType dataType) {
+        AddCollectionFieldReq build = AddCollectionFieldReq.builder().collectionName(collectionWithDynamicField)
+                .dataType(dataType)
+                .fieldName("add_" + CommonData.providerFieldNameByDatatype(dataType))
+                .isNullable(true)
+                .build();
+        boolean vector = dataType.equals(DataType.FloatVector) || dataType.equals(DataType.BinaryVector) || dataType.equals(DataType.SparseFloatVector) || dataType.equals(DataType.Float16Vector) || dataType.equals(DataType.Int8Vector) || dataType.equals(DataType.BFloat16Vector);
+        if (vector) {
+            build.setDimension(CommonData.dim);
+        }
+        if (dataType.equals(DataType.Array)) {
+            build.setElementType(DataType.Int32);
+            build.setMaxCapacity(CommonData.maxCapacity);
+        }
+        try {
+            milvusClientV2.addCollectionField(build);
+            DescribeCollectionResp describeCollectionResp = milvusClientV2.describeCollection(DescribeCollectionReq.builder()
+                    .collectionName(collectionWithDynamicField)
+                    .build());
+            CreateCollectionReq.CollectionSchema collectionSchema = describeCollectionResp.getCollectionSchema();
+            CreateCollectionReq.FieldSchema field = collectionSchema.getField("add_" + CommonData.providerFieldNameByDatatype(dataType));
+            Assert.assertEquals(field.getDataType(), dataType);
+        } catch (Exception e) {
+            Assert.assertTrue(vector);
+            Assert.assertTrue(e.getMessage().contains("not support to add vector field"));
+        }
+    }
+
+    @Test(description = "add field with dynamic name ", groups = {"Smoke"}, dependsOnMethods = {"dynamicFieldCollectionAddField"})
+    public void addFieldWithDynamicName() {
+        milvusClientV2.addCollectionField(AddCollectionFieldReq.builder()
+                .isNullable(true)
+                .dataType(DataType.VarChar)
+                .maxLength(100)
+                .defaultValue("123")
+                .fieldName(CommonData.dynamicField)
+                .collectionName(collectionWithDynamicField)
+                .build());
+        DescribeCollectionResp describeCollectionResp = milvusClientV2.describeCollection(DescribeCollectionReq.builder()
+                .collectionName(collectionWithDynamicField)
+                .build());
+        CreateCollectionReq.CollectionSchema collectionSchema = describeCollectionResp.getCollectionSchema();
+        CreateCollectionReq.FieldSchema field = collectionSchema.getField(CommonData.dynamicField);
+        Assert.assertEquals(field.getDataType(), DataType.VarChar);
+        // 查询
+        List<BaseVector> baseVectors = CommonFunction.providerBaseVector(1, CommonData.dim, DataType.FloatVector);
+        SearchResp search = milvusClientV2.search(SearchReq.builder()
+                .topK(1)
+                .collectionName(collectionWithDynamicField)
+                .outputFields(Lists.newArrayList(CommonData.dynamicField,"$meta"))
+                .data(baseVectors)
+                .consistencyLevel(ConsistencyLevel.BOUNDED)
+                .filter("$meta['dynamicField']['fieldInt64'] < 10")
+                .build());
+        List<SearchResp.SearchResult> searchResults = search.getSearchResults().get(0);
+        Assert.assertEquals(searchResults.size(), 1);
+        SearchResp.SearchResult searchResult = searchResults.get(0);
+        Object o = searchResult.getEntity().get(CommonData.dynamicField);
+        Assert.assertEquals(o.toString(), "123");
+    }
+
+    @Test(description = "add field repeatedly", groups = {"Smoke"}, dependsOnMethods = {"addFieldBeforeLoad"})
+    public void addFieldRepeatedly() {
+        AddCollectionFieldReq build = AddCollectionFieldReq.builder().collectionName(collectionName)
+                .dataType(DataType.VarChar)
+                .fieldName("add_" + CommonData.providerFieldNameByDatatype(DataType.VarChar))
+                .isNullable(true)
+                .build();
+        try {
+            milvusClientV2.addCollectionField(build);
+        } catch (Exception e) {
+            Assert.assertTrue(e.getMessage().contains("duplicate field"));
+        }
+    }
+
+    @Test(description = "add field after alter field name", groups = {"Smoke"}, dependsOnMethods = {"addFieldBeforeLoad"})
+    public void alterFieldNamAfterAddField() {
+        milvusClientV2.alterCollectionField(AlterCollectionFieldReq.builder()
+                .collectionName(collectionName)
+                .fieldName("add_" + CommonData.providerFieldNameByDatatype(DataType.VarChar))
+                .property("max_length", "199")
+                .build());
+        DescribeCollectionResp describeCollectionResp = milvusClientV2.describeCollection(DescribeCollectionReq.builder()
+                .collectionName(collectionName)
+                .build());
+        CreateCollectionReq.CollectionSchema collectionSchema = describeCollectionResp.getCollectionSchema();
+        CreateCollectionReq.FieldSchema field = collectionSchema.getField("add_" + CommonData.providerFieldNameByDatatype(DataType.VarChar));
+        Assert.assertEquals(field.getMaxLength(), 199);
+    }
+
+    @Test(description = "add field after load", groups = {"Smoke"}, dataProvider = "providerDataType")
+    public void addFieldAfterLoad(DataType dataType) {
+        AddCollectionFieldReq build = AddCollectionFieldReq.builder().collectionName(collectionNameWithLoaded)
+                .dataType(dataType)
+                .fieldName("add_" + CommonData.providerFieldNameByDatatype(dataType))
+                .isNullable(true)
+                .build();
+        boolean vector = dataType.equals(DataType.FloatVector) || dataType.equals(DataType.BinaryVector) || dataType.equals(DataType.SparseFloatVector) || dataType.equals(DataType.Float16Vector) || dataType.equals(DataType.Int8Vector) || dataType.equals(DataType.BFloat16Vector);
+        if (vector) {
+            build.setDimension(CommonData.dim);
+        }
+        if (dataType.equals(DataType.Array)) {
+            build.setElementType(DataType.Int32);
+            build.setMaxCapacity(CommonData.maxCapacity);
+        }
+        try {
+            milvusClientV2.addCollectionField(build);
+            DescribeCollectionResp describeCollectionResp = milvusClientV2.describeCollection(DescribeCollectionReq.builder()
+                    .collectionName(collectionNameWithLoaded)
+                    .build());
+            CreateCollectionReq.CollectionSchema collectionSchema = describeCollectionResp.getCollectionSchema();
+            CreateCollectionReq.FieldSchema field = collectionSchema.getField("add_" + CommonData.providerFieldNameByDatatype(dataType));
+            Assert.assertEquals(field.getDataType(), dataType);
+        } catch (Exception e) {
+            Assert.assertTrue(vector);
+            Assert.assertTrue(e.getMessage().contains("not support to add vector field"));
+        }
+    }
+
+
+}
+