Răsfoiți Sursa

getMethod support JSON & SearchResult add method getFieldData() (#519)

xushuang.hu 2 ani în urmă
părinte
comite
e80b04dc8b

+ 92 - 6
examples/main/io/milvus/GeneralExample.java → examples/main/java/io/milvus/GeneralExample.java

@@ -19,7 +19,10 @@
 
 package io.milvus;
 
+import com.alibaba.fastjson.JSONObject;
+import com.google.protobuf.ByteString;
 import io.milvus.client.MilvusServiceClient;
+import io.milvus.common.utils.JacksonUtils;
 import io.milvus.grpc.*;
 import io.milvus.param.*;
 import io.milvus.param.collection.*;
@@ -54,6 +57,7 @@ public class GeneralExample {
     private static final String COLLECTION_NAME = "TEST";
     private static final String ID_FIELD = "userID";
     private static final String VECTOR_FIELD = "userFace";
+    private static final String USER_JSON_FIELD = "userJson";
     private static final Integer VECTOR_DIM = 64;
     private static final String AGE_FIELD = "userAge";
 //    private static final String PROFILE_FIELD = "userProfile";
@@ -66,6 +70,13 @@ public class GeneralExample {
     private static final Integer SEARCH_K = 5;
     private static final String SEARCH_PARAM = "{\"nprobe\":10}";
 
+    private static final String INT32_FIELD_NAME = "int32";
+    private static final String INT64_FIELD_NAME = "int64";
+    private static final String VARCHAR_FIELD_NAME = "varchar";
+    private static final String BOOL_FIELD_NAME = "bool";
+    private static final String FLOAT_FIELD_NAME = "float";
+    private static final String DOUBLE_FIELD_NAME = "double";
+
     private void handleResponseStatus(R<?> r) {
         if (r.getStatus() != R.Status.Success.getCode()) {
             throw new RuntimeException(r.getMessage());
@@ -102,17 +113,25 @@ public class GeneralExample {
 //                .withDimension(BINARY_DIM)
 //                .build();
 
+        FieldType fieldType5 = FieldType.newBuilder()
+                .withName(USER_JSON_FIELD)
+                .withDescription("user json")
+                .withDataType(DataType.JSON)
+                .build();
+
         CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder()
                 .withCollectionName(COLLECTION_NAME)
                 .withDescription("customer info")
                 .withShardsNum(2)
+                .withEnableDynamicField(true)
                 .addFieldType(fieldType1)
                 .addFieldType(fieldType2)
                 .addFieldType(fieldType3)
 //                .addFieldType(fieldType4)
+                .addFieldType(fieldType5)
                 .build();
         R<RpcStatus> response = milvusClient.withTimeout(timeoutMilliseconds, TimeUnit.MILLISECONDS)
-                                            .createCollection(createCollectionReq);
+                .createCollection(createCollectionReq);
         handleResponseStatus(response);
         System.out.println(response);
         return response;
@@ -172,7 +191,7 @@ public class GeneralExample {
         // call flush() to flush the insert buffer to storage,
         // so that the getCollectionStatistics() can get correct number
         milvusClient.flush(FlushParam.newBuilder().addCollectionName(COLLECTION_NAME).build());
-        
+
         System.out.println("========== getCollectionStatistics() ==========");
         R<GetCollectionStatisticsResponse> response = milvusClient.getCollectionStatistics(
                 GetCollectionStatisticsParam.newBuilder()
@@ -421,8 +440,8 @@ public class GeneralExample {
         return response;
     }
 
-    private R<MutationResult> insert(String partitionName, int count) {
-        System.out.println("========== insert() ==========");
+    private R<MutationResult> insertColumns(String partitionName, int count) {
+        System.out.println("========== insertColumns() ==========");
         List<List<Float>> vectors = generateFloatVectors(count);
 //        List<ByteBuffer> profiles = generateBinaryVectors(count);
 
@@ -432,9 +451,18 @@ public class GeneralExample {
             ages.add(ran.nextInt(99));
         }
 
+        List<JSONObject> objectList = new ArrayList<>();
+        for (long i = 0L; i < count ; ++i) {
+            JSONObject obj = new JSONObject();
+            obj.put(INT32_FIELD_NAME, ran.nextInt());
+            obj.put(DOUBLE_FIELD_NAME, ran.nextDouble());
+            objectList.add(obj);
+        }
+
         List<InsertParam.Field> fields = new ArrayList<>();
         fields.add(new InsertParam.Field(AGE_FIELD, ages));
         fields.add(new InsertParam.Field(VECTOR_FIELD, vectors));
+        fields.add(new InsertParam.Field(USER_JSON_FIELD, objectList));
 //        fields.add(new InsertParam.Field(PROFILE_FIELD, profiles));
 
         InsertParam insertParam = InsertParam.newBuilder()
@@ -448,6 +476,46 @@ public class GeneralExample {
         return response;
     }
 
+    private R<MutationResult> insertRows(String partitionName, int count) {
+        System.out.println("========== insertRows() ==========");
+
+        List<JSONObject> rowsData = new ArrayList<>();
+        Random ran = new Random();
+        for (long i = 0L; i < count; ++i) {
+            JSONObject row = new JSONObject();
+            row.put(AGE_FIELD, ran.nextInt(99));
+            row.put(VECTOR_FIELD, generateFloatVector());
+
+            // $meta if collection EnableDynamicField, you can input this field not exist in schema, else deny
+            row.put(INT32_FIELD_NAME, ran.nextInt());
+            row.put(INT64_FIELD_NAME, ran.nextLong());
+            row.put(VARCHAR_FIELD_NAME, "测试varchar");
+            row.put(FLOAT_FIELD_NAME, ran.nextFloat());
+            row.put(DOUBLE_FIELD_NAME, ran.nextDouble());
+            row.put(BOOL_FIELD_NAME, ran.nextBoolean());
+
+            // $json
+            row.put(USER_JSON_FIELD + "[" + INT32_FIELD_NAME + "]", ran.nextInt());
+            row.put(USER_JSON_FIELD + "[" + INT64_FIELD_NAME + "]", ran.nextLong());
+            row.put(USER_JSON_FIELD + "[" + VARCHAR_FIELD_NAME + "]", "测试varchar");
+            row.put(USER_JSON_FIELD + "[" + FLOAT_FIELD_NAME + "]", ran.nextFloat());
+            row.put(USER_JSON_FIELD + "[" + DOUBLE_FIELD_NAME + "]", ran.nextDouble());
+            row.put(USER_JSON_FIELD + "[" + BOOL_FIELD_NAME + "]", ran.nextBoolean());
+
+            rowsData.add(row);
+        }
+
+        InsertParam insertParam = InsertParam.newBuilder()
+                .withCollectionName(COLLECTION_NAME)
+                .withPartitionName(partitionName)
+                .withRows(rowsData)
+                .build();
+
+        R<MutationResult> response = milvusClient.insert(insertParam);
+        handleResponseStatus(response);
+        return response;
+    }
+
     private List<List<Float>> generateFloatVectors(int count) {
         Random ran = new Random();
         List<List<Float>> vectors = new ArrayList<>();
@@ -462,6 +530,15 @@ public class GeneralExample {
         return vectors;
     }
 
+    private List<Float> generateFloatVector() {
+        Random ran = new Random();
+        List<Float> vector = new ArrayList<>();
+        for (int i = 0; i < VECTOR_DIM; ++i) {
+            vector.add(ran.nextFloat());
+        }
+        return vector;
+    }
+
 //    private List<ByteBuffer> generateBinaryVectors(int count) {
 //        Random ran = new Random();
 //        List<ByteBuffer> vectors = new ArrayList<>();
@@ -496,11 +573,20 @@ public class GeneralExample {
         List<Long> deleteIds = new ArrayList<>();
         Random ran = new Random();
         for (int i = 0; i < 100; ++i) {
-            R<MutationResult> result = example.insert(partitionName, row_count);
+            // insertColumns
+            R<MutationResult> result = example.insertColumns(partitionName, row_count);
             MutationResultWrapper wrapper = new MutationResultWrapper(result.getData());
             List<Long> ids = wrapper.getLongIDs();
             deleteIds.add(ids.get(ran.nextInt(row_count)));
         }
+
+        // insertRows
+        R<MutationResult> result = example.insertRows(partitionName, row_count);
+        MutationResultWrapper wrapper = new MutationResultWrapper(result.getData());
+        long insertCount = wrapper.getInsertCount();
+        List<Long> longIDs = wrapper.getLongIDs();
+        System.out.println("complete insertRows, insertCount:" + insertCount + "; longIDs:" + longIDs);
+
         example.getCollectionStatistics();
 
         // Must create an index before load(), FLAT is brute-force search(no index)
@@ -553,4 +639,4 @@ public class GeneralExample {
         example.dropIndex();
         example.dropCollection();
     }
-}
+}

+ 119 - 0
examples/main/java/io/milvus/RBACExample.java

@@ -0,0 +1,119 @@
+package io.milvus;
+
+import io.milvus.client.MilvusServiceClient;
+import io.milvus.grpc.ListCredUsersResponse;
+import io.milvus.grpc.SelectRoleResponse;
+import io.milvus.param.ConnectParam;
+import io.milvus.param.R;
+import io.milvus.param.RpcStatus;
+import io.milvus.param.credential.CreateCredentialParam;
+import io.milvus.param.credential.ListCredUsersParam;
+import io.milvus.param.role.*;
+import org.apache.commons.lang3.Validate;
+
+public class RBACExample {
+    private static final MilvusServiceClient milvusClient;
+
+    static {
+        ConnectParam connectParam = ConnectParam.newBuilder()
+                .withHost("localhost")
+                .withPort(19530)
+                .withAuthorization("root","Milvus")
+                .build();
+        milvusClient = new MilvusServiceClient(connectParam);
+    }
+
+    public static R<RpcStatus> createUser(String userName, String password) {
+        return milvusClient.createCredential(CreateCredentialParam.newBuilder()
+                .withUsername(userName)
+                .withPassword(password)
+                .build());
+    }
+
+    public static R<RpcStatus> grantUserRole(String userName, String roleName) {
+        return milvusClient.addUserToRole(AddUserToRoleParam.newBuilder()
+                .withUserName(userName)
+                .withRoleName(roleName)
+                .build());
+    }
+
+    public static R<RpcStatus> revokeUserRole(String userName, String roleName) {
+        return milvusClient.removeUserFromRole(RemoveUserFromRoleParam.newBuilder()
+                .withUserName(userName)
+                .withRoleName(roleName)
+                .build());
+    }
+
+    public static R<ListCredUsersResponse> listUsers() {
+        return milvusClient.listCredUsers(ListCredUsersParam.newBuilder()
+                .build());
+    }
+
+    public static R<SelectRoleResponse> selectRole(String roleName) {
+        return milvusClient.selectRole(SelectRoleParam.newBuilder()
+                .withRoleName(roleName)
+                .build());
+    }
+
+    public static R<RpcStatus> createRole(String roleName) {
+        return milvusClient.createRole(CreateRoleParam.newBuilder()
+                .withRoleName(roleName)
+                .build());
+    }
+
+    public static R<RpcStatus> dropRole(String roleName) {
+        return milvusClient.dropRole(DropRoleParam.newBuilder()
+                .withRoleName(roleName)
+                .build());
+    }
+
+    public static R<RpcStatus> grantRolePrivilege(String roleName, String objectType, String objectName, String privilege) {
+        return milvusClient.grantRolePrivilege(GrantRolePrivilegeParam.newBuilder()
+                .withRoleName(roleName)
+                .withObject(objectType)
+                .withObjectName(objectName)
+                .withPrivilege(privilege)
+                .build());
+    }
+
+    public static R<RpcStatus> revokeRolePrivilege(String roleName, String objectType, String objectName, String privilege) {
+        return milvusClient.revokeRolePrivilege(RevokeRolePrivilegeParam.newBuilder()
+                .withRoleName(roleName)
+                .withObject(objectType)
+                .withObjectName(objectName)
+                .withPrivilege(privilege)
+                .build());
+    }
+
+
+    public static void main(String[] args) {
+        // create a role
+        R<RpcStatus> resp = createRole("role1");
+        Validate.isTrue(resp.getStatus() == R.success().getStatus(), "create role fail!");
+
+        //create user
+        resp = createUser("user", "pwd123456");
+        Validate.isTrue(resp.getStatus() == R.success().getStatus(), "create user fail!");
+
+        // grant privilege to role.
+        // grant object is all collections, grant object type is Collection, and the privilege is CreateCollection
+        resp = grantRolePrivilege("role1","Global","*",  "CreateCollection");
+        Validate.isTrue(resp.getStatus() == R.success().getStatus(), "bind privileges to role fail!");
+
+        // bind role to user
+        resp = grantUserRole("user", "role1");
+        Validate.isTrue(resp.getStatus() == R.success().getStatus(), "bind role to user fail!");
+
+        // revoke privilege from role
+        resp = revokeRolePrivilege("role1","Global","*",  "CreateCollection");
+        Validate.isTrue(resp.getStatus() == R.success().getStatus(), "revoke privileges to role fail!");
+
+        // list role
+        R<SelectRoleResponse> resp1 = selectRole("role1");
+        Validate.isTrue(resp1.getStatus() == R.success().getStatus(), "select role information fail!");
+
+        // delete a role
+        resp = dropRole("role1");
+        Validate.isTrue(resp.getStatus() == R.success().getStatus(), "drop role fail!");
+    }
+}

+ 1 - 1
src/main/java/io/milvus/param/ParamUtils.java

@@ -148,7 +148,7 @@ public class ParamUtils {
                 break;
             case JSON:
                 for (Object value : values) {
-                    if (!(value instanceof ByteString)) {
+                    if (!(value instanceof JSONObject)) {
                         throw new ParamException(String.format(errMsgs.get(dataType), fieldSchema.getName()));
                     }
                 }

+ 10 - 10
src/main/java/io/milvus/response/FieldDataWrapper.java

@@ -175,43 +175,43 @@ public class FieldDataWrapper {
     }
 
     public Integer getAsInt(int index, String paramName) {
-        if (isDynamicField()) {
+        if (isJsonField()) {
             String result = getAsString(index, paramName);
             return result == null ? null : Integer.parseInt(result);
         }
-        throw new IllegalResponseException("The field is not dynamic and does not support this operation");
+        throw new IllegalResponseException("Only JSON type support this operation");
     }
 
     public String getAsString(int index, String paramName) {
-        if (isDynamicField()) {
+        if (isJsonField()) {
             JSONObject jsonObject = parseObjectData(index);
             return jsonObject.getString(paramName);
         }
-        throw new IllegalResponseException("The field is not dynamic and does not support this operation");
+        throw new IllegalResponseException("Only JSON type support this operation");
     }
 
     public Boolean getAsBool(int index, String paramName) {
-        if (isDynamicField()) {
+        if (isJsonField()) {
             String result = getAsString(index, paramName);
             return result == null ? null : Boolean.parseBoolean(result);
         }
-        throw new IllegalResponseException("The field is not dynamic and does not support this operation");
+        throw new IllegalResponseException("Only JSON type support this operation");
     }
 
     public Double getAsDouble(int index, String paramName) {
-        if (isDynamicField()) {
+        if (isJsonField()) {
             String result = getAsString(index, paramName);
             return result == null ? null : Double.parseDouble(result);
         }
-        throw new IllegalResponseException("The field is not dynamic and does not support this operation");
+        throw new IllegalResponseException("Only JSON type support this operation");
     }
 
     public Object get(int index, String paramName) throws Exception {
-        if (isDynamicField()) {
+        if (isJsonField()) {
             JSONObject jsonObject = parseObjectData(index);
             return jsonObject.get(paramName);
         }
-        throw new IllegalResponseException("The field is not dynamic and does not support this operation");
+        throw new IllegalResponseException("Only JSON type support this operation");
     }
 
     private Object valueByIdx(int index) {

+ 18 - 0
src/main/java/io/milvus/response/SearchResultsWrapper.java

@@ -19,6 +19,24 @@ public class SearchResultsWrapper {
         this.results = results;
     }
 
+    /**
+     * Gets {@link FieldDataWrapper} for a field.
+     * Throws {@link ParamException} if the field doesn't exist.
+     *
+     * @param fieldName field name to get output data
+     * @return {@link FieldDataWrapper}
+     */
+    public FieldDataWrapper getFieldWrapper(@NonNull String fieldName) throws ParamException {
+        List<FieldData> fields = results.getFieldsDataList();
+        for (FieldData field : fields) {
+            if (fieldName.compareTo(field.getFieldName()) == 0) {
+                return new FieldDataWrapper(field);
+            }
+        }
+
+        throw new ParamException("The field name doesn't exist");
+    }
+
     /**
      * Gets data for an output field which is specified by search request.
      * Throws {@link ParamException} if the field doesn't exist.