Support function/analyzer (#1175)

Signed-off-by: yhmo <yihua.mo@zilliz.com>

src/main/java/io/milvus/common/clientenum/FunctionType.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.milvus.common.clientenum;
+
+public enum FunctionType {
+    UNKNOWN,
+    BM25,
+}

src/main/java/io/milvus/v2/common/IndexParam.java

@@ -55,6 +55,9 @@ public class IndexParam {
         // Only for binary vectors
         HAMMING,
         JACCARD,
+
+        // Only for sparse vector with BM25
+        BM25,
         ;
     }
 

src/main/java/io/milvus/v2/service/collection/CollectionService.java

@@ -33,6 +33,7 @@ import io.milvus.v2.utils.SchemaUtils;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -110,19 +111,27 @@ public class CollectionService extends BaseService {
         String title = String.format("CreateCollectionRequest collectionName:%s", request.getCollectionName());
 
         //convert CollectionSchema to io.milvus.grpc.CollectionSchema
-        CollectionSchema grpcSchema = CollectionSchema.newBuilder()
+        CollectionSchema.Builder grpcSchemaBuilder = CollectionSchema.newBuilder()
                 .setName(request.getCollectionName())
                 .setDescription(request.getDescription())
-                .setEnableDynamicField(request.getCollectionSchema().isEnableDynamicField())
-                .build();
+                .setEnableDynamicField(request.getCollectionSchema().isEnableDynamicField());
+        List<String> outputFields = new ArrayList<>();
+        for (CreateCollectionReq.Function function : request.getCollectionSchema().getFunctionList()) {
+            grpcSchemaBuilder.addFunctions(SchemaUtils.convertToGrpcFunction(function)).build();
+            outputFields.addAll(function.getOutputFieldNames());
+        }
         for (CreateCollectionReq.FieldSchema fieldSchema : request.getCollectionSchema().getFieldSchemaList()) {
-            grpcSchema = grpcSchema.toBuilder().addFields(SchemaUtils.convertToGrpcFieldSchema(fieldSchema)).build();
+            FieldSchema grpcFieldSchema = SchemaUtils.convertToGrpcFieldSchema(fieldSchema);
+            if (outputFields.contains(fieldSchema.getName())) {
+                grpcFieldSchema = grpcFieldSchema.toBuilder().setIsFunctionOutput(true).build();
+            }
+            grpcSchemaBuilder.addFields(grpcFieldSchema);
         }
 
         //create collection
         CreateCollectionRequest.Builder builder = CreateCollectionRequest.newBuilder()
                 .setCollectionName(request.getCollectionName())
-                .setSchema(grpcSchema.toByteString())
+                .setSchema(grpcSchemaBuilder.build().toByteString())
                 .setShardsNum(request.getNumShards());
         List<KeyValuePair> propertiesList = ParamUtils.AssembleKvPair(request.getProperties());
         if (CollectionUtils.isNotEmpty(propertiesList)) {

src/main/java/io/milvus/v2/service/collection/request/AddFieldReq.java

@@ -24,6 +24,8 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.experimental.SuperBuilder;
 
+import java.util.Map;
+
 @Data
 @SuperBuilder
 public class AddFieldReq {
@@ -50,6 +52,9 @@ public class AddFieldReq {
     private Object defaultValue = null; // only for scalar fields
     @Builder.ObtainVia(field = "hiddenField")
     private boolean enableDefaultValue = false; // a flag to pass the default value to server or not
+    private Boolean enableAnalyzer; // for BM25 tokenizer
+    Map<String, Object> analyzerParams; // for BM25 tokenizer
+    private Boolean enableMatch; // for BM25 keyword search
 
     AddFieldReq setDefaultValue(Object obj) {
         enableDefaultValue = true; // automatically set this flag

src/main/java/io/milvus/v2/service/collection/request/CreateCollectionReq.java

@@ -19,6 +19,7 @@
 
 package io.milvus.v2.service.collection.request;
 
+import io.milvus.common.clientenum.FunctionType;
 import io.milvus.v2.common.ConsistencyLevel;
 import io.milvus.v2.common.DataType;
 import io.milvus.v2.common.IndexParam;
@@ -129,6 +130,8 @@ public class CreateCollectionReq {
         private List<CreateCollectionReq.FieldSchema> fieldSchemaList = new ArrayList<>();
         @Builder.Default
         private boolean enableDynamicField = false;
+        @Builder.Default
+        private List<CreateCollectionReq.Function> functionList = new ArrayList<>();
 
         public CollectionSchema addField(AddFieldReq addFieldReq) {
             // check the input here to pop error messages earlier
@@ -148,6 +151,9 @@ public class CreateCollectionReq {
                     .autoID(addFieldReq.getAutoID())
                     .isNullable(addFieldReq.getIsNullable())
                     .defaultValue(addFieldReq.getDefaultValue())
+                    .enableAnalyzer(addFieldReq.getEnableAnalyzer())
+                    .enableMatch(addFieldReq.getEnableMatch())
+                    .analyzerParams(addFieldReq.getAnalyzerParams())
                     .build();
             if (addFieldReq.getDataType().equals(DataType.Array)) {
                 if (addFieldReq.getElementType() == null) {
@@ -168,6 +174,11 @@ public class CreateCollectionReq {
             return this;
         }
 
+        public CollectionSchema addFunction(Function function) {
+            functionList.add(function);
+            return this;
+        }
+
         public CreateCollectionReq.FieldSchema getField(String fieldName) {
             for (CreateCollectionReq.FieldSchema field : fieldSchemaList) {
                 if (field.getName().equals(fieldName)) {
@@ -202,5 +213,21 @@ public class CreateCollectionReq {
         private Boolean isNullable = Boolean.FALSE; // only for scalar fields(not include Array fields)
         @Builder.Default
         private Object defaultValue = null; // only for scalar fields
+        private Boolean enableAnalyzer; // for BM25 tokenizer
+        Map<String, Object> analyzerParams; // for BM25 tokenizer
+        private Boolean enableMatch; // for BM25 keyword search
+    }
+
+    @Data
+    @SuperBuilder
+    public static class Function {
+        private String name;
+        @Builder.Default
+        private String description = "";
+        FunctionType functionType;
+        @Builder.Default
+        List<String> inputFieldNames = new ArrayList<>();
+        @Builder.Default
+        List<String> outputFieldNames = new ArrayList<>();
     }
 }

src/main/java/io/milvus/v2/utils/SchemaUtils.java

@@ -19,17 +19,27 @@
 
 package io.milvus.v2.utils;
 
+import com.google.gson.reflect.TypeToken;
+import io.milvus.common.utils.JsonUtils;
 import io.milvus.grpc.*;
 import io.milvus.param.ParamUtils;
 import io.milvus.v2.exception.ErrorCode;
 import io.milvus.v2.exception.MilvusClientException;
 import io.milvus.v2.service.collection.request.CreateCollectionReq;
+import org.apache.commons.lang3.StringUtils;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
+import java.util.stream.Collectors;
 
 public class SchemaUtils {
+    public static void checkNullEmptyString(String target, String title) {
+        if (target == null || StringUtils.isBlank(target)) {
+            throw new MilvusClientException(ErrorCode.INVALID_PARAMS, title + " cannot be null or empty");
+        }
+    }
     public static FieldSchema convertToGrpcFieldSchema(CreateCollectionReq.FieldSchema fieldSchema) {
+        checkNullEmptyString(fieldSchema.getName(), "Field name");
+
         DataType dType = DataType.valueOf(fieldSchema.getDataType().name());
         FieldSchema.Builder builder = FieldSchema.newBuilder()
                 .setName(fieldSchema.getName())
@@ -52,24 +62,52 @@ public class SchemaUtils {
             }
         }
 
-        FieldSchema schema = builder.build();
         if(fieldSchema.getDimension() != null){
-            schema = schema.toBuilder().addTypeParams(KeyValuePair.newBuilder().setKey("dim").setValue(String.valueOf(fieldSchema.getDimension())).build()).build();
+            builder.addTypeParams(KeyValuePair.newBuilder().setKey("dim").setValue(String.valueOf(fieldSchema.getDimension())).build()).build();
         }
 //        if (Objects.equals(fieldSchema.getName(), partitionKeyField)) {
 //            schema = schema.toBuilder().setIsPartitionKey(Boolean.TRUE).build();
 //        }
         if(fieldSchema.getDataType() == io.milvus.v2.common.DataType.VarChar && fieldSchema.getMaxLength() != null){
-            schema = schema.toBuilder().addTypeParams(KeyValuePair.newBuilder().setKey("max_length").setValue(String.valueOf(fieldSchema.getMaxLength())).build()).build();
+            builder.addTypeParams(KeyValuePair.newBuilder().setKey("max_length").setValue(String.valueOf(fieldSchema.getMaxLength())).build()).build();
         }
         if (fieldSchema.getDataType() == io.milvus.v2.common.DataType.Array) {
-            schema = schema.toBuilder().addTypeParams(KeyValuePair.newBuilder().setKey("max_capacity").setValue(String.valueOf(fieldSchema.getMaxCapacity())).build()).build();
-            schema = schema.toBuilder().setElementType(DataType.valueOf(fieldSchema.getElementType().name())).build();
+            builder.addTypeParams(KeyValuePair.newBuilder().setKey("max_capacity").setValue(String.valueOf(fieldSchema.getMaxCapacity())).build()).build();
+            builder.setElementType(DataType.valueOf(fieldSchema.getElementType().name())).build();
             if (fieldSchema.getElementType() == io.milvus.v2.common.DataType.VarChar && fieldSchema.getMaxLength() != null) {
-                schema = schema.toBuilder().addTypeParams(KeyValuePair.newBuilder().setKey("max_length").setValue(String.valueOf(fieldSchema.getMaxLength())).build()).build();
+                builder.addTypeParams(KeyValuePair.newBuilder().setKey("max_length").setValue(String.valueOf(fieldSchema.getMaxLength())).build()).build();
             }
         }
-        return schema;
+
+        if (fieldSchema.getEnableAnalyzer() != null) {
+            builder.addTypeParams(KeyValuePair.newBuilder().setKey("enable_analyzer").setValue(String.valueOf(fieldSchema.getEnableAnalyzer())).build()).build();
+        }
+        if (fieldSchema.getEnableMatch() != null) {
+            builder.addTypeParams(KeyValuePair.newBuilder().setKey("enable_match").setValue(String.valueOf(fieldSchema.getEnableMatch())).build()).build();
+        }
+        if (fieldSchema.getAnalyzerParams() != null) {
+            String params = JsonUtils.toJson(fieldSchema.getAnalyzerParams());
+            builder.addTypeParams(KeyValuePair.newBuilder().setKey("analyzer_params").setValue(params).build()).build();
+        }
+        return builder.build();
+    }
+
+    public static FunctionSchema convertToGrpcFunction(CreateCollectionReq.Function function) {
+        checkNullEmptyString(function.getName(), "Function name");
+
+        FunctionSchema.Builder builder = FunctionSchema.newBuilder()
+                .setName(function.getName())
+                .setDescription(function.getDescription())
+                .setType(FunctionType.valueOf(function.getFunctionType().name()));
+
+        for (String name : function.getInputFieldNames()) {
+            builder.addInputFieldNames(name);
+        }
+        for (String name : function.getOutputFieldNames()) {
+            builder.addOutputFieldNames(name);
+        }
+
+        return builder.build();
     }
 
     public static CreateCollectionReq.CollectionSchema convertFromGrpcCollectionSchema(CollectionSchema schema) {
@@ -80,6 +118,13 @@ public class SchemaUtils {
             fieldSchemas.add(convertFromGrpcFieldSchema(fieldSchema));
         }
         collectionSchema.setFieldSchemaList(fieldSchemas);
+
+        List<CreateCollectionReq.Function> functions = new ArrayList<>();
+        for (FunctionSchema functionSchema : schema.getFunctionsList()) {
+            functions.add(convertFromGrpcFunction(functionSchema));
+        }
+        collectionSchema.setFunctionList(functions);
+
         return collectionSchema;
     }
 
@@ -103,8 +148,26 @@ public class SchemaUtils {
                 schema.setMaxLength(Integer.parseInt(keyValuePair.getValue()));
             } else if(keyValuePair.getKey().equals("max_capacity")){
                 schema.setMaxCapacity(Integer.parseInt(keyValuePair.getValue()));
+            } else if(keyValuePair.getKey().equals("enable_analyzer")){
+                schema.setEnableAnalyzer(Boolean.parseBoolean(keyValuePair.getValue()));
+            } else if(keyValuePair.getKey().equals("enable_match")){
+                schema.setEnableMatch(Boolean.parseBoolean(keyValuePair.getValue()));
+            } else if(keyValuePair.getKey().equals("analyzer_params")){
+                Map<String, Object> params = JsonUtils.fromJson(keyValuePair.getValue(), new TypeToken<Map<String, Object>>() {}.getType());
+                schema.setAnalyzerParams(params);
             }
         }
         return schema;
     }
+
+    public static CreateCollectionReq.Function convertFromGrpcFunction(FunctionSchema functionSchema) {
+        CreateCollectionReq.Function function = CreateCollectionReq.Function.builder()
+                .name(functionSchema.getName())
+                .description(functionSchema.getDescription())
+                .functionType(io.milvus.common.clientenum.FunctionType.valueOf(functionSchema.getType().name()))
+                .inputFieldNames(functionSchema.getInputFieldNamesList().stream().collect(Collectors.toList()))
+                .outputFieldNames(functionSchema.getOutputFieldNamesList().stream().collect(Collectors.toList()))
+                .build();
+        return function;
+    }
 }

src/main/java/io/milvus/v2/utils/VectorUtils.java

@@ -19,6 +19,7 @@
 
 package io.milvus.v2.utils;
 
+import com.google.gson.JsonElement;
 import com.google.protobuf.ByteString;
 import io.milvus.common.utils.GTsDict;
 import io.milvus.common.utils.JsonUtils;
@@ -227,54 +228,99 @@ public class VectorUtils {
         return builder.build();
     }
 
+    private static TemplateArrayValue deduceTemplateArray(List<?> array) {
+        if (array.isEmpty()) {
+            return TemplateArrayValue.newBuilder().build(); // an empty list?
+        }
+
+        Object firstObj = array.get(0);
+        if (firstObj instanceof Boolean) {
+            BoolArray.Builder builder = BoolArray.newBuilder();
+            array.forEach(val->{
+                if (!(val instanceof Boolean)) {
+                    throw new ParamException("Filter expression template is a list, the first value is Boolean, but some elements are not Boolean");
+                }
+                builder.addData((Boolean)val);
+            });
+            return TemplateArrayValue.newBuilder().setBoolData(builder.build()).build();
+        } else if (firstObj instanceof Integer || firstObj instanceof Long) {
+            LongArray.Builder builder = LongArray.newBuilder();
+            array.forEach(val->{
+                if (!(val instanceof Integer) && !(val instanceof Long)) {
+                    throw new ParamException("Filter expression template is a list, the first value is Integer/Long, but some elements are not Integer/Long");
+                }
+                builder.addData((val instanceof Integer) ? (Integer)val : (Long)val);
+            });
+            return TemplateArrayValue.newBuilder().setLongData(builder.build()).build();
+        } else if (firstObj instanceof Double) {
+            DoubleArray.Builder builder = DoubleArray.newBuilder();
+            array.forEach(val->{
+                if (!(val instanceof Double)) {
+                    throw new ParamException("Filter expression template is a list, the first value is Double, but some elements are not Double");
+                }
+                builder.addData((Double)val);
+            });
+            return TemplateArrayValue.newBuilder().setDoubleData(builder.build()).build();
+        } else if (firstObj instanceof String) {
+            StringArray.Builder builder = StringArray.newBuilder();
+            array.forEach(val->{
+                if (!(val instanceof String)) {
+                    throw new ParamException("Filter expression template is a list, the first value is String, but some elements are not String");
+                }
+                builder.addData((String)val);
+            });
+            return TemplateArrayValue.newBuilder().setStringData(builder.build()).build();
+        } else if (firstObj instanceof JsonElement) {
+            JSONArray.Builder builder = JSONArray.newBuilder();
+            array.forEach(val->{
+                if (!(val instanceof JsonElement)) {
+                    throw new ParamException("Filter expression template is a list, the first value is JsonElement, but some elements are not JsonElement");
+                }
+                String str = JsonUtils.toJson((JsonElement)val);
+                builder.addData(ByteString.copyFromUtf8(str));
+            });
+            return TemplateArrayValue.newBuilder().setJsonData(builder.build()).build();
+        } else if (firstObj instanceof List) {
+            TemplateArrayValueArray.Builder builder = TemplateArrayValueArray.newBuilder();
+            array.forEach(val->{
+                if (!(val instanceof List)) {
+                    throw new ParamException("Filter expression template is a list, the first value is List, but some elements are not List");
+                }
+                List<?> subArrary = (List<?>)val;
+                builder.addData(deduceTemplateArray(subArrary));
+            });
+
+            return TemplateArrayValue.newBuilder().setArrayData(builder.build()).build();
+        } else {
+            throw new ParamException("Unsupported value type for filter expression template.");
+        }
+    }
+
     public static TemplateValue deduceAndCreateTemplateValue(Object value) {
         if (value instanceof Boolean) {
             return TemplateValue.newBuilder()
                     .setBoolVal((Boolean)value)
-                    .setType(DataType.Bool)
                     .build();
-        } else if (value instanceof Integer) {
+        } else if (value instanceof Integer || value instanceof Long) {
            return TemplateValue.newBuilder()
-                   .setInt64Val((Integer)value)
-                   .setType(DataType.Int64)
+                   .setInt64Val((value instanceof Integer) ? (Integer)value : (Long)value)
                    .build();
         } else if (value instanceof Double) {
             return TemplateValue.newBuilder()
                     .setFloatVal((Double)value)
-                    .setType(DataType.Double)
                     .build();
         } else if (value instanceof String) {
             return TemplateValue.newBuilder()
                     .setStringVal((String)value)
-                    .setType(DataType.VarChar)
                     .build();
         } else if (value instanceof List) {
-            // TemplateArrayValue and TemplateValue can nest each other
-            // The element_type of TemplateArrayValue is deduced by its elements:
-            //   1. if all the elements are the same type, element_type is the first element's type
-            //   2. if not the same type, element_type is DataType.JSON
-            List<Object> array = (List<Object>)value;
-            TemplateArrayValue.Builder builder = TemplateArrayValue.newBuilder();
-            DataType lastType = DataType.UNRECOGNIZED;
-            boolean sameType = true;
-            for (Object obj : array) {
-                TemplateValue tv = deduceAndCreateTemplateValue(obj);
-                builder.addArray(tv);
-                if (sameType && lastType != DataType.UNRECOGNIZED && lastType != tv.getType()) {
-                    sameType = false;
-                }
-                lastType = tv.getType();
-            }
-            DataType arrayType = sameType ? lastType : DataType.JSON;
-            builder.setElementType(arrayType);
-            builder.setSameType(sameType);
-
+            List<?> array = (List<?>)value;
+            TemplateArrayValue tav = deduceTemplateArray(array);
             return TemplateValue.newBuilder()
-                    .setType(DataType.Array)
-                    .setArrayVal(builder.build())
+                    .setArrayVal(tav)
                     .build();
         } else {
-            throw new ParamException("Unsupported value type for expression template.");
+            throw new ParamException("Unsupported value type for filter expression template.");
         }
     }
 

src/main/milvus-proto

@@ -1 +1 @@
-Subproject commit 4d5c88b00cf7280b17542940d3b49041f1c92f66
+Subproject commit 806678b2806ed7ba10dd44ed4a31ea7492448849

src/test/java/io/milvus/client/MilvusClientDockerTest.java

@@ -83,7 +83,7 @@ class MilvusClientDockerTest {
     private static final Random RANDOM = new Random();
 
     @Container
-    private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:master-20240927-1f271e39-amd64");
+    private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:master-20241111-fca946de-amd64");
 
     @BeforeAll
     public static void setUp() {
@@ -2885,136 +2885,136 @@ class MilvusClientDockerTest {
         }
     }
 
-    @Test
-    public void testIterator() {
-        String randomCollectionName = generator.generate(10);
-
-        CollectionSchemaParam schema = buildSchema(true, false, true,
-                Arrays.asList(DataType.FloatVector, DataType.JSON));
-
-        // create collection
-        CreateCollectionParam createParam = CreateCollectionParam.newBuilder()
-                .withCollectionName(randomCollectionName)
-                .withSchema(schema)
-                .build();
-
-        R<RpcStatus> createR = client.createCollection(createParam);
-        Assertions.assertEquals(R.Status.Success.getCode(), createR.getStatus().intValue());
-
-        // insert data
-        int rowCount = 1000;
-        List<JsonObject> rows = new ArrayList<>();
-        for (long i = 0L; i < rowCount; ++i) {
-            JsonObject row = new JsonObject();
-            row.addProperty("id", Long.toString(i));
-            row.add(DataType.FloatVector.name(), JsonUtils.toJsonTree(generateFloatVectors(1).get(0)));
-            JsonObject json = new JsonObject();
-            if (i%2 == 0) {
-                json.addProperty("even", true);
-            }
-            row.add(DataType.JSON.name(), json);
-            row.addProperty("dynamic", i);
-            rows.add(row);
-        }
-
-        InsertParam insertParam = InsertParam.newBuilder()
-                .withCollectionName(randomCollectionName)
-                .withRows(rows)
-                .build();
-
-        R<MutationResult> insertR = client.insert(insertParam);
-        Assertions.assertEquals(R.Status.Success.getCode(), insertR.getStatus().intValue());
-
-        // create index
-        CreateIndexParam indexParam = CreateIndexParam.newBuilder()
-                .withCollectionName(randomCollectionName)
-                .withFieldName(DataType.FloatVector.name())
-                .withIndexType(IndexType.FLAT)
-                .withMetricType(MetricType.L2)
-                .build();
-
-        R<RpcStatus> createIndexR = client.createIndex(indexParam);
-        Assertions.assertEquals(R.Status.Success.getCode(), createIndexR.getStatus().intValue());
-
-        // load collection
-        R<RpcStatus> loadR = client.loadCollection(LoadCollectionParam.newBuilder()
-                .withCollectionName(randomCollectionName)
-                .build());
-        Assertions.assertEquals(R.Status.Success.getCode(), loadR.getStatus().intValue());
-
-        // query iterator
-        QueryIteratorParam.Builder queryIteratorParamBuilder = QueryIteratorParam.newBuilder()
-                .withCollectionName(randomCollectionName)
-                .withExpr("dynamic < 300")
-                .withOutFields(Lists.newArrayList("*"))
-                .withBatchSize(100L)
-                .withConsistencyLevel(ConsistencyLevelEnum.BOUNDED);
-
-        R<QueryIterator> qResponse = client.queryIterator(queryIteratorParamBuilder.build());
-        Assertions.assertEquals(R.Status.Success.getCode(), qResponse.getStatus().intValue());
-
-        QueryIterator queryIterator = qResponse.getData();
-        int counter = 0;
-        while (true) {
-            List<QueryResultsWrapper.RowRecord> res = queryIterator.next();
-            if (res.isEmpty()) {
-                System.out.println("query iteration finished, close");
-                queryIterator.close();
-                break;
-            }
-
-            for (QueryResultsWrapper.RowRecord record : res) {
-                Assertions.assertInstanceOf(Long.class, record.get("dynamic"));
-                Assertions.assertInstanceOf(String.class, record.get("id"));
-                Object vec = record.get(DataType.FloatVector.name());
-                Assertions.assertInstanceOf(List.class, vec);
-                List<Float> vector = (List<Float>)vec;
-                Assertions.assertEquals(DIMENSION, vector.size());
-                Assertions.assertInstanceOf(JsonElement.class, record.get(DataType.JSON.name()));
-//                System.out.println(record);
-                counter++;
-            }
-        }
-        Assertions.assertEquals(300, counter);
-
-        // search iterator
-        List<List<Float>> vectors = generateFloatVectors(1);
-        SearchIteratorParam.Builder searchIteratorParamBuilder = SearchIteratorParam.newBuilder()
-                .withCollectionName(randomCollectionName)
-                .withOutFields(Lists.newArrayList("*"))
-                .withBatchSize(10L)
-                .withVectorFieldName(DataType.FloatVector.name())
-                .withFloatVectors(vectors)
-                .withTopK(50)
-                .withMetricType(MetricType.L2);
-
-        R<SearchIterator> sResponse = client.searchIterator(searchIteratorParamBuilder.build());
-        Assertions.assertEquals(R.Status.Success.getCode(), sResponse.getStatus().intValue());
-
-        SearchIterator searchIterator = sResponse.getData();
-        counter = 0;
-        while (true) {
-            List<QueryResultsWrapper.RowRecord> res = searchIterator.next();
-            if (res.isEmpty()) {
-                System.out.println("search iteration finished, close");
-                searchIterator.close();
-                break;
-            }
-
-            for (QueryResultsWrapper.RowRecord record : res) {
-                Assertions.assertInstanceOf(Long.class, record.get("dynamic"));
-                Assertions.assertInstanceOf(String.class, record.get("id"));
-                Object vec = record.get(DataType.FloatVector.name());
-                Assertions.assertInstanceOf(List.class, vec);
-                List<Float> vector = (List<Float>)vec;
-                Assertions.assertEquals(DIMENSION, vector.size());
-                Assertions.assertInstanceOf(JsonElement.class, record.get(DataType.JSON.name()));
-//                System.out.println(record);
-                counter++;
-            }
-        }
-        Assertions.assertEquals(50, counter);
-    }
+//    @Test
+//    public void testIterator() {
+//        String randomCollectionName = generator.generate(10);
+//
+//        CollectionSchemaParam schema = buildSchema(true, false, true,
+//                Arrays.asList(DataType.FloatVector, DataType.JSON));
+//
+//        // create collection
+//        CreateCollectionParam createParam = CreateCollectionParam.newBuilder()
+//                .withCollectionName(randomCollectionName)
+//                .withSchema(schema)
+//                .build();
+//
+//        R<RpcStatus> createR = client.createCollection(createParam);
+//        Assertions.assertEquals(R.Status.Success.getCode(), createR.getStatus().intValue());
+//
+//        // insert data
+//        int rowCount = 1000;
+//        List<JsonObject> rows = new ArrayList<>();
+//        for (long i = 0L; i < rowCount; ++i) {
+//            JsonObject row = new JsonObject();
+//            row.addProperty("id", Long.toString(i));
+//            row.add(DataType.FloatVector.name(), JsonUtils.toJsonTree(generateFloatVectors(1).get(0)));
+//            JsonObject json = new JsonObject();
+//            if (i%2 == 0) {
+//                json.addProperty("even", true);
+//            }
+//            row.add(DataType.JSON.name(), json);
+//            row.addProperty("dynamic", i);
+//            rows.add(row);
+//        }
+//
+//        InsertParam insertParam = InsertParam.newBuilder()
+//                .withCollectionName(randomCollectionName)
+//                .withRows(rows)
+//                .build();
+//
+//        R<MutationResult> insertR = client.insert(insertParam);
+//        Assertions.assertEquals(R.Status.Success.getCode(), insertR.getStatus().intValue());
+//
+//        // create index
+//        CreateIndexParam indexParam = CreateIndexParam.newBuilder()
+//                .withCollectionName(randomCollectionName)
+//                .withFieldName(DataType.FloatVector.name())
+//                .withIndexType(IndexType.FLAT)
+//                .withMetricType(MetricType.L2)
+//                .build();
+//
+//        R<RpcStatus> createIndexR = client.createIndex(indexParam);
+//        Assertions.assertEquals(R.Status.Success.getCode(), createIndexR.getStatus().intValue());
+//
+//        // load collection
+//        R<RpcStatus> loadR = client.loadCollection(LoadCollectionParam.newBuilder()
+//                .withCollectionName(randomCollectionName)
+//                .build());
+//        Assertions.assertEquals(R.Status.Success.getCode(), loadR.getStatus().intValue());
+//
+//        // query iterator
+//        QueryIteratorParam.Builder queryIteratorParamBuilder = QueryIteratorParam.newBuilder()
+//                .withCollectionName(randomCollectionName)
+//                .withExpr("dynamic < 300")
+//                .withOutFields(Lists.newArrayList("*"))
+//                .withBatchSize(100L)
+//                .withConsistencyLevel(ConsistencyLevelEnum.BOUNDED);
+//
+//        R<QueryIterator> qResponse = client.queryIterator(queryIteratorParamBuilder.build());
+//        Assertions.assertEquals(R.Status.Success.getCode(), qResponse.getStatus().intValue());
+//
+//        QueryIterator queryIterator = qResponse.getData();
+//        int counter = 0;
+//        while (true) {
+//            List<QueryResultsWrapper.RowRecord> res = queryIterator.next();
+//            if (res.isEmpty()) {
+//                System.out.println("query iteration finished, close");
+//                queryIterator.close();
+//                break;
+//            }
+//
+//            for (QueryResultsWrapper.RowRecord record : res) {
+//                Assertions.assertInstanceOf(Long.class, record.get("dynamic"));
+//                Assertions.assertInstanceOf(String.class, record.get("id"));
+//                Object vec = record.get(DataType.FloatVector.name());
+//                Assertions.assertInstanceOf(List.class, vec);
+//                List<Float> vector = (List<Float>)vec;
+//                Assertions.assertEquals(DIMENSION, vector.size());
+//                Assertions.assertInstanceOf(JsonElement.class, record.get(DataType.JSON.name()));
+////                System.out.println(record);
+//                counter++;
+//            }
+//        }
+//        Assertions.assertEquals(300, counter);
+//
+//        // search iterator
+//        List<List<Float>> vectors = generateFloatVectors(1);
+//        SearchIteratorParam.Builder searchIteratorParamBuilder = SearchIteratorParam.newBuilder()
+//                .withCollectionName(randomCollectionName)
+//                .withOutFields(Lists.newArrayList("*"))
+//                .withBatchSize(10L)
+//                .withVectorFieldName(DataType.FloatVector.name())
+//                .withFloatVectors(vectors)
+//                .withTopK(50)
+//                .withMetricType(MetricType.L2);
+//
+//        R<SearchIterator> sResponse = client.searchIterator(searchIteratorParamBuilder.build());
+//        Assertions.assertEquals(R.Status.Success.getCode(), sResponse.getStatus().intValue());
+//
+//        SearchIterator searchIterator = sResponse.getData();
+//        counter = 0;
+//        while (true) {
+//            List<QueryResultsWrapper.RowRecord> res = searchIterator.next();
+//            if (res.isEmpty()) {
+//                System.out.println("search iteration finished, close");
+//                searchIterator.close();
+//                break;
+//            }
+//
+//            for (QueryResultsWrapper.RowRecord record : res) {
+//                Assertions.assertInstanceOf(Long.class, record.get("dynamic"));
+//                Assertions.assertInstanceOf(String.class, record.get("id"));
+//                Object vec = record.get(DataType.FloatVector.name());
+//                Assertions.assertInstanceOf(List.class, vec);
+//                List<Float> vector = (List<Float>)vec;
+//                Assertions.assertEquals(DIMENSION, vector.size());
+//                Assertions.assertInstanceOf(JsonElement.class, record.get(DataType.JSON.name()));
+////                System.out.println(record);
+//                counter++;
+//            }
+//        }
+//        Assertions.assertEquals(50, counter);
+//    }
 
     @Test
     void testDatabase() {

src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java

@@ -23,6 +23,7 @@ import com.google.common.collect.Lists;
 import com.google.gson.*;
 
 import com.google.gson.reflect.TypeToken;
+import io.milvus.common.clientenum.FunctionType;
 import io.milvus.common.utils.Float16Utils;
 import io.milvus.common.utils.JsonUtils;
 import io.milvus.orm.iterator.QueryIterator;
@@ -70,7 +71,7 @@ class MilvusClientV2DockerTest {
     private static final Random RANDOM = new Random();
 
     @Container
-    private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:master-20240927-1f271e39-amd64");
+    private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:master-20241111-fca946de-amd64");
 
     @BeforeAll
     public static void setUp() {
@@ -1386,213 +1387,213 @@ class MilvusClientV2DockerTest {
         Assertions.assertEquals(1L, insertResp.getInsertCnt());
     }
 
-    @Test
-    public void testIterator() {
-        String randomCollectionName = generator.generate(10);
-        CreateCollectionReq.CollectionSchema collectionSchema = baseSchema();
-        collectionSchema.addField(AddFieldReq.builder()
-                .fieldName("float_vector")
-                .dataType(DataType.FloatVector)
-                .dimension(dimension)
-                .build());
-        collectionSchema.addField(AddFieldReq.builder()
-                .fieldName("binary_vector")
-                .dataType(DataType.BinaryVector)
-                .dimension(dimension)
-                .build());
-        collectionSchema.addField(AddFieldReq.builder()
-                .fieldName("sparse_vector")
-                .dataType(DataType.SparseFloatVector)
-                .dimension(dimension)
-                .build());
-        collectionSchema.addField(AddFieldReq.builder()
-                .fieldName("bfloat16_vector")
-                .dataType(DataType.BFloat16Vector)
-                .dimension(dimension)
-                .build());
-
-        List<IndexParam> indexParams = new ArrayList<>();
-        indexParams.add(IndexParam.builder()
-                .fieldName("float_vector")
-                .indexType(IndexParam.IndexType.FLAT)
-                .metricType(IndexParam.MetricType.L2)
-                .build());
-        indexParams.add(IndexParam.builder()
-                .fieldName("binary_vector")
-                .indexType(IndexParam.IndexType.BIN_FLAT)
-                .metricType(IndexParam.MetricType.HAMMING)
-                .build());
-        indexParams.add(IndexParam.builder()
-                .fieldName("sparse_vector")
-                .indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
-                .metricType(IndexParam.MetricType.IP)
-                .extraParams(new HashMap<String,Object>(){{put("drop_ratio_build", 0.1);}})
-                .build());
-        indexParams.add(IndexParam.builder()
-                .fieldName("bfloat16_vector")
-                .indexType(IndexParam.IndexType.FLAT)
-                .metricType(IndexParam.MetricType.COSINE)
-                .build());
-
-        CreateCollectionReq requestCreate = CreateCollectionReq.builder()
-                .collectionName(randomCollectionName)
-                .collectionSchema(collectionSchema)
-                .indexParams(indexParams)
-                .build();
-        client.createCollection(requestCreate);
-
-        // insert rows
-        long count = 10000;
-        List<JsonObject> data = generateRandomData(collectionSchema, count);
-        InsertResp insertResp = client.insert(InsertReq.builder()
-                .collectionName(randomCollectionName)
-                .data(data)
-                .build());
-        Assertions.assertEquals(count, insertResp.getInsertCnt());
-
-        // get row count
-        long rowCount = getRowCount(randomCollectionName);
-        Assertions.assertEquals(count, rowCount);
-
-        // search iterator
-        SearchIterator searchIterator = client.searchIterator(SearchIteratorReq.builder()
-                .collectionName(randomCollectionName)
-                .outputFields(Lists.newArrayList("*"))
-                .batchSize(20L)
-                .vectorFieldName("float_vector")
-                .vectors(Collections.singletonList(new FloatVec(generateFolatVector())))
-                .expr("int64_field > 500 && int64_field < 1000")
-                .params("{\"range_filter\": 5.0, \"radius\": 50.0}")
-                .topK(1000)
-                .metricType(IndexParam.MetricType.L2)
-                .consistencyLevel(ConsistencyLevel.EVENTUALLY)
-                .build());
-
-        int counter = 0;
-        while (true) {
-            List<QueryResultsWrapper.RowRecord> res = searchIterator.next();
-            if (res.isEmpty()) {
-                System.out.println("search iteration finished, close");
-                searchIterator.close();
-                break;
-            }
-
-            for (QueryResultsWrapper.RowRecord record : res) {
-                Assertions.assertInstanceOf(Float.class, record.get("score"));
-                Assertions.assertTrue((float)record.get("score") >= 5.0);
-                Assertions.assertTrue((float)record.get("score") <= 50.0);
-
-                Assertions.assertInstanceOf(Boolean.class, record.get("bool_field"));
-                Assertions.assertInstanceOf(Integer.class, record.get("int8_field"));
-                Assertions.assertInstanceOf(Integer.class, record.get("int16_field"));
-                Assertions.assertInstanceOf(Integer.class, record.get("int32_field"));
-                Assertions.assertInstanceOf(Long.class, record.get("int64_field"));
-                Assertions.assertInstanceOf(Float.class, record.get("float_field"));
-                Assertions.assertInstanceOf(Double.class, record.get("double_field"));
-                Assertions.assertInstanceOf(String.class, record.get("varchar_field"));
-                Assertions.assertInstanceOf(JsonObject.class, record.get("json_field"));
-                Assertions.assertInstanceOf(List.class, record.get("arr_int_field"));
-                Assertions.assertInstanceOf(List.class, record.get("float_vector"));
-                Assertions.assertInstanceOf(ByteBuffer.class, record.get("binary_vector"));
-                Assertions.assertInstanceOf(ByteBuffer.class, record.get("bfloat16_vector"));
-                Assertions.assertInstanceOf(SortedMap.class, record.get("sparse_vector"));
-
-                long int64Val = (long)record.get("int64_field");
-                Assertions.assertTrue(int64Val > 500L && int64Val < 1000L);
-
-                String varcharVal = (String)record.get("varchar_field");
-                Assertions.assertTrue(varcharVal.startsWith("varchar_"));
-
-                JsonObject jsonObj = (JsonObject)record.get("json_field");
-                Assertions.assertTrue(jsonObj.has(String.format("JSON_%d", int64Val)));
-
-                List<Integer> intArr = (List<Integer>)record.get("arr_int_field");
-                Assertions.assertTrue(intArr.size() <= 50); // max capacity 50 is defined in the baseSchema()
-
-                List<Float> floatVector = (List<Float>)record.get("float_vector");
-                Assertions.assertEquals(dimension, floatVector.size());
-
-                ByteBuffer binaryVector = (ByteBuffer)record.get("binary_vector");
-                Assertions.assertEquals(dimension, binaryVector.limit()*8);
-
-                ByteBuffer bfloat16Vector = (ByteBuffer)record.get("bfloat16_vector");
-                Assertions.assertEquals(dimension*2, bfloat16Vector.limit());
-
-                SortedMap<Long, Float> sparseVector = (SortedMap<Long, Float>)record.get("sparse_vector");
-                Assertions.assertTrue(sparseVector.size() >= 10 && sparseVector.size() <= 20); // defined in generateSparseVector()
-
-                counter++;
-            }
-        }
-        System.out.println(String.format("There are %d items match score between [5.0, 50.0]", counter));
-
-        // query iterator
-        QueryIterator queryIterator = client.queryIterator(QueryIteratorReq.builder()
-                .collectionName(randomCollectionName)
-                .expr("int64_field < 300")
-                .outputFields(Lists.newArrayList("*"))
-                .batchSize(50L)
-                .offset(5)
-                .limit(400)
-                .consistencyLevel(ConsistencyLevel.EVENTUALLY)
-                .build());
-
-        counter = 0;
-        while (true) {
-            List<QueryResultsWrapper.RowRecord> res = queryIterator.next();
-            if (res.isEmpty()) {
-                System.out.println("query iteration finished, close");
-                queryIterator.close();
-                break;
-            }
-
-            for (QueryResultsWrapper.RowRecord record : res) {
-                Assertions.assertInstanceOf(Boolean.class, record.get("bool_field"));
-                Assertions.assertInstanceOf(Integer.class, record.get("int8_field"));
-                Assertions.assertInstanceOf(Integer.class, record.get("int16_field"));
-                Assertions.assertInstanceOf(Integer.class, record.get("int32_field"));
-                Assertions.assertInstanceOf(Long.class, record.get("int64_field"));
-                Assertions.assertInstanceOf(Float.class, record.get("float_field"));
-                Assertions.assertInstanceOf(Double.class, record.get("double_field"));
-                Assertions.assertInstanceOf(String.class, record.get("varchar_field"));
-                Assertions.assertInstanceOf(JsonObject.class, record.get("json_field"));
-                Assertions.assertInstanceOf(List.class, record.get("arr_int_field"));
-                Assertions.assertInstanceOf(List.class, record.get("float_vector"));
-                Assertions.assertInstanceOf(ByteBuffer.class, record.get("binary_vector"));
-                Assertions.assertInstanceOf(ByteBuffer.class, record.get("bfloat16_vector"));
-                Assertions.assertInstanceOf(SortedMap.class, record.get("sparse_vector"));
-
-                long int64Val = (long)record.get("int64_field");
-                Assertions.assertTrue(int64Val < 300L);
-
-                String varcharVal = (String)record.get("varchar_field");
-                Assertions.assertTrue(varcharVal.startsWith("varchar_"));
-
-                JsonObject jsonObj = (JsonObject)record.get("json_field");
-                Assertions.assertTrue(jsonObj.has(String.format("JSON_%d", int64Val)));
-
-                List<Integer> intArr = (List<Integer>)record.get("arr_int_field");
-                Assertions.assertTrue(intArr.size() <= 50); // max capacity 50 is defined in the baseSchema()
-
-                List<Float> floatVector = (List<Float>)record.get("float_vector");
-                Assertions.assertEquals(dimension, floatVector.size());
-
-                ByteBuffer binaryVector = (ByteBuffer)record.get("binary_vector");
-                Assertions.assertEquals(dimension, binaryVector.limit()*8);
-
-                ByteBuffer bfloat16Vector = (ByteBuffer)record.get("bfloat16_vector");
-                Assertions.assertEquals(dimension*2, bfloat16Vector.limit());
-
-                SortedMap<Long, Float> sparseVector = (SortedMap<Long, Float>)record.get("sparse_vector");
-                Assertions.assertTrue(sparseVector.size() >= 10 && sparseVector.size() <= 20); // defined in generateSparseVector()
-
-                counter++;
-            }
-        }
-        Assertions.assertEquals(295, counter);
-
-        client.dropCollection(DropCollectionReq.builder().collectionName(randomCollectionName).build());
-    }
+//    @Test
+//    public void testIterator() {
+//        String randomCollectionName = generator.generate(10);
+//        CreateCollectionReq.CollectionSchema collectionSchema = baseSchema();
+//        collectionSchema.addField(AddFieldReq.builder()
+//                .fieldName("float_vector")
+//                .dataType(DataType.FloatVector)
+//                .dimension(dimension)
+//                .build());
+//        collectionSchema.addField(AddFieldReq.builder()
+//                .fieldName("binary_vector")
+//                .dataType(DataType.BinaryVector)
+//                .dimension(dimension)
+//                .build());
+//        collectionSchema.addField(AddFieldReq.builder()
+//                .fieldName("sparse_vector")
+//                .dataType(DataType.SparseFloatVector)
+//                .dimension(dimension)
+//                .build());
+//        collectionSchema.addField(AddFieldReq.builder()
+//                .fieldName("bfloat16_vector")
+//                .dataType(DataType.BFloat16Vector)
+//                .dimension(dimension)
+//                .build());
+//
+//        List<IndexParam> indexParams = new ArrayList<>();
+//        indexParams.add(IndexParam.builder()
+//                .fieldName("float_vector")
+//                .indexType(IndexParam.IndexType.FLAT)
+//                .metricType(IndexParam.MetricType.L2)
+//                .build());
+//        indexParams.add(IndexParam.builder()
+//                .fieldName("binary_vector")
+//                .indexType(IndexParam.IndexType.BIN_FLAT)
+//                .metricType(IndexParam.MetricType.HAMMING)
+//                .build());
+//        indexParams.add(IndexParam.builder()
+//                .fieldName("sparse_vector")
+//                .indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
+//                .metricType(IndexParam.MetricType.IP)
+//                .extraParams(new HashMap<String,Object>(){{put("drop_ratio_build", 0.1);}})
+//                .build());
+//        indexParams.add(IndexParam.builder()
+//                .fieldName("bfloat16_vector")
+//                .indexType(IndexParam.IndexType.FLAT)
+//                .metricType(IndexParam.MetricType.COSINE)
+//                .build());
+//
+//        CreateCollectionReq requestCreate = CreateCollectionReq.builder()
+//                .collectionName(randomCollectionName)
+//                .collectionSchema(collectionSchema)
+//                .indexParams(indexParams)
+//                .build();
+//        client.createCollection(requestCreate);
+//
+//        // insert rows
+//        long count = 10000;
+//        List<JsonObject> data = generateRandomData(collectionSchema, count);
+//        InsertResp insertResp = client.insert(InsertReq.builder()
+//                .collectionName(randomCollectionName)
+//                .data(data)
+//                .build());
+//        Assertions.assertEquals(count, insertResp.getInsertCnt());
+//
+//        // get row count
+//        long rowCount = getRowCount(randomCollectionName);
+//        Assertions.assertEquals(count, rowCount);
+//
+//        // search iterator
+//        SearchIterator searchIterator = client.searchIterator(SearchIteratorReq.builder()
+//                .collectionName(randomCollectionName)
+//                .outputFields(Lists.newArrayList("*"))
+//                .batchSize(20L)
+//                .vectorFieldName("float_vector")
+//                .vectors(Collections.singletonList(new FloatVec(generateFolatVector())))
+//                .expr("int64_field > 500 && int64_field < 1000")
+//                .params("{\"range_filter\": 5.0, \"radius\": 50.0}")
+//                .topK(1000)
+//                .metricType(IndexParam.MetricType.L2)
+//                .consistencyLevel(ConsistencyLevel.EVENTUALLY)
+//                .build());
+//
+//        int counter = 0;
+//        while (true) {
+//            List<QueryResultsWrapper.RowRecord> res = searchIterator.next();
+//            if (res.isEmpty()) {
+//                System.out.println("search iteration finished, close");
+//                searchIterator.close();
+//                break;
+//            }
+//
+//            for (QueryResultsWrapper.RowRecord record : res) {
+//                Assertions.assertInstanceOf(Float.class, record.get("score"));
+//                Assertions.assertTrue((float)record.get("score") >= 5.0);
+//                Assertions.assertTrue((float)record.get("score") <= 50.0);
+//
+//                Assertions.assertInstanceOf(Boolean.class, record.get("bool_field"));
+//                Assertions.assertInstanceOf(Integer.class, record.get("int8_field"));
+//                Assertions.assertInstanceOf(Integer.class, record.get("int16_field"));
+//                Assertions.assertInstanceOf(Integer.class, record.get("int32_field"));
+//                Assertions.assertInstanceOf(Long.class, record.get("int64_field"));
+//                Assertions.assertInstanceOf(Float.class, record.get("float_field"));
+//                Assertions.assertInstanceOf(Double.class, record.get("double_field"));
+//                Assertions.assertInstanceOf(String.class, record.get("varchar_field"));
+//                Assertions.assertInstanceOf(JsonObject.class, record.get("json_field"));
+//                Assertions.assertInstanceOf(List.class, record.get("arr_int_field"));
+//                Assertions.assertInstanceOf(List.class, record.get("float_vector"));
+//                Assertions.assertInstanceOf(ByteBuffer.class, record.get("binary_vector"));
+//                Assertions.assertInstanceOf(ByteBuffer.class, record.get("bfloat16_vector"));
+//                Assertions.assertInstanceOf(SortedMap.class, record.get("sparse_vector"));
+//
+//                long int64Val = (long)record.get("int64_field");
+//                Assertions.assertTrue(int64Val > 500L && int64Val < 1000L);
+//
+//                String varcharVal = (String)record.get("varchar_field");
+//                Assertions.assertTrue(varcharVal.startsWith("varchar_"));
+//
+//                JsonObject jsonObj = (JsonObject)record.get("json_field");
+//                Assertions.assertTrue(jsonObj.has(String.format("JSON_%d", int64Val)));
+//
+//                List<Integer> intArr = (List<Integer>)record.get("arr_int_field");
+//                Assertions.assertTrue(intArr.size() <= 50); // max capacity 50 is defined in the baseSchema()
+//
+//                List<Float> floatVector = (List<Float>)record.get("float_vector");
+//                Assertions.assertEquals(dimension, floatVector.size());
+//
+//                ByteBuffer binaryVector = (ByteBuffer)record.get("binary_vector");
+//                Assertions.assertEquals(dimension, binaryVector.limit()*8);
+//
+//                ByteBuffer bfloat16Vector = (ByteBuffer)record.get("bfloat16_vector");
+//                Assertions.assertEquals(dimension*2, bfloat16Vector.limit());
+//
+//                SortedMap<Long, Float> sparseVector = (SortedMap<Long, Float>)record.get("sparse_vector");
+//                Assertions.assertTrue(sparseVector.size() >= 10 && sparseVector.size() <= 20); // defined in generateSparseVector()
+//
+//                counter++;
+//            }
+//        }
+//        System.out.println(String.format("There are %d items match score between [5.0, 50.0]", counter));
+//
+//        // query iterator
+//        QueryIterator queryIterator = client.queryIterator(QueryIteratorReq.builder()
+//                .collectionName(randomCollectionName)
+//                .expr("int64_field < 300")
+//                .outputFields(Lists.newArrayList("*"))
+//                .batchSize(50L)
+//                .offset(5)
+//                .limit(400)
+//                .consistencyLevel(ConsistencyLevel.EVENTUALLY)
+//                .build());
+//
+//        counter = 0;
+//        while (true) {
+//            List<QueryResultsWrapper.RowRecord> res = queryIterator.next();
+//            if (res.isEmpty()) {
+//                System.out.println("query iteration finished, close");
+//                queryIterator.close();
+//                break;
+//            }
+//
+//            for (QueryResultsWrapper.RowRecord record : res) {
+//                Assertions.assertInstanceOf(Boolean.class, record.get("bool_field"));
+//                Assertions.assertInstanceOf(Integer.class, record.get("int8_field"));
+//                Assertions.assertInstanceOf(Integer.class, record.get("int16_field"));
+//                Assertions.assertInstanceOf(Integer.class, record.get("int32_field"));
+//                Assertions.assertInstanceOf(Long.class, record.get("int64_field"));
+//                Assertions.assertInstanceOf(Float.class, record.get("float_field"));
+//                Assertions.assertInstanceOf(Double.class, record.get("double_field"));
+//                Assertions.assertInstanceOf(String.class, record.get("varchar_field"));
+//                Assertions.assertInstanceOf(JsonObject.class, record.get("json_field"));
+//                Assertions.assertInstanceOf(List.class, record.get("arr_int_field"));
+//                Assertions.assertInstanceOf(List.class, record.get("float_vector"));
+//                Assertions.assertInstanceOf(ByteBuffer.class, record.get("binary_vector"));
+//                Assertions.assertInstanceOf(ByteBuffer.class, record.get("bfloat16_vector"));
+//                Assertions.assertInstanceOf(SortedMap.class, record.get("sparse_vector"));
+//
+//                long int64Val = (long)record.get("int64_field");
+//                Assertions.assertTrue(int64Val < 300L);
+//
+//                String varcharVal = (String)record.get("varchar_field");
+//                Assertions.assertTrue(varcharVal.startsWith("varchar_"));
+//
+//                JsonObject jsonObj = (JsonObject)record.get("json_field");
+//                Assertions.assertTrue(jsonObj.has(String.format("JSON_%d", int64Val)));
+//
+//                List<Integer> intArr = (List<Integer>)record.get("arr_int_field");
+//                Assertions.assertTrue(intArr.size() <= 50); // max capacity 50 is defined in the baseSchema()
+//
+//                List<Float> floatVector = (List<Float>)record.get("float_vector");
+//                Assertions.assertEquals(dimension, floatVector.size());
+//
+//                ByteBuffer binaryVector = (ByteBuffer)record.get("binary_vector");
+//                Assertions.assertEquals(dimension, binaryVector.limit()*8);
+//
+//                ByteBuffer bfloat16Vector = (ByteBuffer)record.get("bfloat16_vector");
+//                Assertions.assertEquals(dimension*2, bfloat16Vector.limit());
+//
+//                SortedMap<Long, Float> sparseVector = (SortedMap<Long, Float>)record.get("sparse_vector");
+//                Assertions.assertTrue(sparseVector.size() >= 10 && sparseVector.size() <= 20); // defined in generateSparseVector()
+//
+//                counter++;
+//            }
+//        }
+//        Assertions.assertEquals(295, counter);
+//
+//        client.dropCollection(DropCollectionReq.builder().collectionName(randomCollectionName).build());
+//    }
 
     @Test
     void testDatabase() {
@@ -1976,4 +1977,88 @@ class MilvusClientV2DockerTest {
             System.out.println(result);
         }
     }
+
+    @Test
+    void testDocInOut() {
+        String randomCollectionName = generator.generate(10);
+
+        CreateCollectionReq.CollectionSchema collectionSchema = CreateCollectionReq.CollectionSchema.builder()
+                .build();
+        collectionSchema.addField(AddFieldReq.builder()
+                .fieldName("id")
+                .dataType(DataType.Int64)
+                .isPrimaryKey(Boolean.TRUE)
+                .autoID(Boolean.FALSE)
+                .build());
+        collectionSchema.addField(AddFieldReq.builder()
+                .fieldName("dense")
+                .dataType(DataType.FloatVector)
+                .dimension(dimension)
+                .build());
+        collectionSchema.addField(AddFieldReq.builder()
+                .fieldName("sparse")
+                .dataType(DataType.SparseFloatVector)
+                .build());
+        Map<String, Object> analyzerParams = new HashMap<>();
+        analyzerParams.put("tokenizer", "standard");
+        collectionSchema.addField(AddFieldReq.builder()
+                .fieldName("text")
+                .dataType(DataType.VarChar)
+                .maxLength(100)
+                .enableAnalyzer(true)
+                .enableMatch(true)
+                .analyzerParams(analyzerParams)
+                .build());
+
+        collectionSchema.addFunction(CreateCollectionReq.Function.builder()
+                .name("bm25")
+                .description("desc bm25")
+                .functionType(FunctionType.BM25)
+                .inputFieldNames(Collections.singletonList("text"))
+                .outputFieldNames(Collections.singletonList("sparse"))
+                .build());
+
+        List<IndexParam> indexParams = new ArrayList<>();
+        indexParams.add(IndexParam.builder()
+                .fieldName("dense")
+                .indexType(IndexParam.IndexType.FLAT)
+                .metricType(IndexParam.MetricType.L2)
+                .build());
+        indexParams.add(IndexParam.builder()
+                .fieldName("sparse")
+                .indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
+                .metricType(IndexParam.MetricType.BM25)
+                .extraParams(new HashMap<String,Object>(){{put("drop_ratio_build", 0.1);}})
+                .build());
+        CreateCollectionReq requestCreate = CreateCollectionReq.builder()
+                .collectionName(randomCollectionName)
+                .collectionSchema(collectionSchema)
+                .indexParams(indexParams)
+                .build();
+        client.createCollection(requestCreate);
+        System.out.println("Collection created");
+
+        DescribeCollectionResp descResp = client.describeCollection(DescribeCollectionReq.builder()
+                .collectionName(randomCollectionName)
+                .build());
+
+        CreateCollectionReq.CollectionSchema collSchema = descResp.getCollectionSchema();
+        CreateCollectionReq.FieldSchema fieldSchema = collSchema.getField("text");
+        Assertions.assertNotNull(fieldSchema);
+        Assertions.assertTrue(fieldSchema.getEnableAnalyzer());
+        Assertions.assertTrue(fieldSchema.getEnableAnalyzer());
+        Map<String, Object> params = fieldSchema.getAnalyzerParams();
+        Assertions.assertTrue(params.containsKey("tokenizer"));
+        Assertions.assertEquals("standard", params.get("tokenizer"));
+
+        List<CreateCollectionReq.Function> functions = collSchema.getFunctionList();
+        Assertions.assertEquals(1, functions.size());
+        Assertions.assertEquals("bm25", functions.get(0).getName());
+        Assertions.assertEquals("desc bm25", functions.get(0).getDescription());
+        Assertions.assertEquals(FunctionType.BM25, functions.get(0).getFunctionType());
+        Assertions.assertEquals(1, functions.get(0).getInputFieldNames().size());
+        Assertions.assertEquals("text", functions.get(0).getInputFieldNames().get(0));
+        Assertions.assertEquals(1, functions.get(0).getOutputFieldNames().size());
+        Assertions.assertEquals("sparse", functions.get(0).getOutputFieldNames().get(0));
+    }
 }