Flatten search params (#1246)

Signed-off-by: yhmo <yihua.mo@zilliz.com>

sdk-core/src/main/java/io/milvus/param/Constant.java

@@ -36,6 +36,7 @@ public class Constant {
     public static final String GROUP_BY_FIELD = "group_by_field";
     public static final String GROUP_SIZE = "group_size";
     public static final String STRICT_GROUP_SIZE = "strict_group_size";
+    public static final String HINTS = "hints"; // from milvus v2.5
 
     public static final String INDEX_TYPE = "index_type";
     public static final String METRIC_TYPE = "metric_type";

sdk-core/src/main/java/io/milvus/param/ParamUtils.java

@@ -766,6 +766,82 @@ public class ParamUtils {
         return placeholderGroup.toByteString();
     }
 
+    // in versions older than milvus v2.5.1, the search parameters are organized as:
+    //    search_params
+    //    {
+    //        "topk"
+    //        "round_decimal"
+    //        "ignore_growing"
+    //        "offset"
+    //        "metric_type"
+    //        "group_by_field"
+    //        "group_size"
+    //        "strict_group_size"
+    //
+    //        "params": {
+    //            "nprobe"
+    //            "ef"
+    //            "reorder_k"
+    //            "max_empty_result_buckets"
+    //            "drop_ratio_search"
+    //            "radius"
+    //            "range_filter"
+    //        }
+    //    }
+    // in milvus v2.5.1, the server only parse parameters from top level.
+    // to compatible with old versions, we make a copy for all the parameters into the "params",
+    // so that both old milvus and new milvus can read parameters
+    //    search_params
+    //    {
+    //        "topk": "10"
+    //        "round_decimal": "6"
+    //        "ignore_growing": "true"
+    //        "offset": "0"
+    //        "metric_type": "L2"
+    //        "group_by_field": "aaa"
+    //        "group_size": "10"
+    //        "strict_group_size": "5"
+    //
+    //        "nprobe": "16"
+    //        "reorder_k": "5"
+    //        "ef": "32"
+    //        "drop_ratio_search": "0.2"
+    //        "max_empty_result_buckets": "2"
+    //        "radius": "0.5"
+    //        "range_filter": "0.8"
+    //
+    //        "params": {
+    //            "nprobe": 16,
+    //            "reorder_k": 5,
+    //            "ef": 32,
+    //            "drop_ratio_search": 0.2,
+    //            "max_empty_result_buckets": 2,
+    //            "radius": 0.5,
+    //            "range_filter": 0.8
+    //        }
+    //    }
+    // the following logic tries to fit the compatibility between v2.5.1 and older versions
+    public static void compatibleSearchParams(Map<String, Object> searchParams, SearchRequest.Builder builder) {
+        searchParams.forEach((key, value) -> {
+            // for new versions, all keys are in the top level
+            builder.addSearchParams(
+                    KeyValuePair.newBuilder()
+                            .setKey(key)
+                            .setValue(String.valueOf(value))
+                            .build());
+        });
+        try {
+            String params = JsonUtils.toJson(searchParams);
+            builder.addSearchParams(
+                    KeyValuePair.newBuilder()
+                            .setKey(Constant.PARAMS)
+                            .setValue(params)
+                            .build());
+        } catch (IllegalArgumentException e) {
+            throw new MilvusClientException(ErrorCode.INVALID_PARAMS, e.getMessage() + e.getCause().getMessage());
+        }
+    }
+
     @SuppressWarnings("unchecked")
     public static SearchRequest convertSearchParam(@NonNull SearchParam requestParam) throws ParamException {
         SearchRequest.Builder builder = SearchRequest.newBuilder()
@@ -784,6 +860,26 @@ public class ParamUtils {
         builder.setNq(requestParam.getNQ());
 
         // search parameters
+        // tries to fit the compatibility between v2.5.1 and older versions
+        if (null != requestParam.getParams() && !requestParam.getParams().isEmpty()) {
+            try {
+                Map<String, Object> paramMap = JsonUtils.fromJson(requestParam.getParams(),
+                        new TypeToken<Map<String, Object>>() {}.getType());
+                compatibleSearchParams(paramMap, builder);
+
+                String offset = paramMap.getOrDefault(Constant.OFFSET, 0).toString();
+                builder.addSearchParams(
+                        KeyValuePair.newBuilder()
+                                .setKey(Constant.OFFSET)
+                                .setValue(offset)
+                                .build());
+            } catch (IllegalArgumentException e) {
+                throw new ParamException(e.getMessage() + e.getCause().getMessage());
+            }
+        }
+
+        // the following parameters are not changed
+        // just note: if the searchParams already contains the same key, the following parameters will overwrite it
         builder.addSearchParams(
                 KeyValuePair.newBuilder()
                         .setKey(Constant.VECTOR_FIELD)
@@ -835,26 +931,6 @@ public class ParamUtils {
             }
         }
 
-        if (null != requestParam.getParams() && !requestParam.getParams().isEmpty()) {
-            try {
-                Map<String, Object> paramMap = JsonUtils.fromJson(requestParam.getParams(),
-                        new TypeToken<Map<String, Object>>() {}.getType());
-                String offset = paramMap.getOrDefault(Constant.OFFSET, 0).toString();
-                builder.addSearchParams(
-                        KeyValuePair.newBuilder()
-                                .setKey(Constant.OFFSET)
-                                .setValue(offset)
-                                .build());
-                builder.addSearchParams(
-                        KeyValuePair.newBuilder()
-                                .setKey(Constant.PARAMS)
-                                .setValue(requestParam.getParams())
-                                .build());
-            } catch (IllegalArgumentException e) {
-                throw new ParamException(e.getMessage() + e.getCause().getMessage());
-            }
-        }
-
         if (!requestParam.getOutFields().isEmpty()) {
             requestParam.getOutFields().forEach(builder::addOutputFields);
         }
@@ -886,6 +962,14 @@ public class ParamUtils {
         builder.setPlaceholderGroup(byteStr);
         builder.setNq(annSearchParam.getNQ());
 
+        // search parameters
+        // tries to fit the compatibility between v2.5.1 and older versions
+        Map<String, Object> paramMap = new HashMap<>();
+        if (null != annSearchParam.getParams() && !annSearchParam.getParams().isEmpty()) {
+            paramMap = JsonUtils.fromJson(annSearchParam.getParams(), new TypeToken<Map<String, Object>>() {}.getType());
+        }
+        ParamUtils.compatibleSearchParams(paramMap, builder);
+
         builder.addSearchParams(
                         KeyValuePair.newBuilder()
                                 .setKey(Constant.VECTOR_FIELD)
@@ -905,16 +989,6 @@ public class ParamUtils {
                             .build());
         }
 
-        // params
-        String params = "{}";
-        if (null != annSearchParam.getParams() && !annSearchParam.getParams().isEmpty()) {
-            params = annSearchParam.getParams();
-        }
-        builder.addSearchParams(KeyValuePair.newBuilder()
-                .setKey(Constant.PARAMS)
-                .setValue(params)
-                .build());
-
         // always use expression since dsl is discarded
         builder.setDslType(DslType.BoolExprV1);
         if (annSearchParam.getExpr() != null && !annSearchParam.getExpr().isEmpty()) {

sdk-core/src/main/java/io/milvus/v2/utils/VectorUtils.java

@@ -20,6 +20,7 @@
 package io.milvus.v2.utils;
 
 import com.google.gson.JsonElement;
+import com.google.gson.reflect.TypeToken;
 import com.google.protobuf.ByteString;
 import io.milvus.common.utils.GTsDict;
 import io.milvus.common.utils.JsonUtils;
@@ -161,6 +162,12 @@ public class VectorUtils {
         builder.setNq(vectors.size());
 
         // search parameters
+        // tries to fit the compatibility between v2.5.1 and older versions
+        Map<String, Object> searchParams = request.getSearchParams();
+        ParamUtils.compatibleSearchParams(searchParams, builder);
+
+        // the following parameters are not changed
+        // just note: if the searchParams already contains the same key, the following parameters will overwrite it
         if (StringUtils.isNotEmpty(request.getAnnsField())) {
             builder.addSearchParams(
                     KeyValuePair.newBuilder()
@@ -198,19 +205,6 @@ public class VectorUtils {
                             .build());
         }
 
-        if (null != request.getSearchParams()) {
-            try {
-                String searchParams = JsonUtils.toJson(request.getSearchParams());
-                builder.addSearchParams(
-                        KeyValuePair.newBuilder()
-                                .setKey(Constant.PARAMS)
-                                .setValue(searchParams)
-                                .build());
-            } catch (IllegalArgumentException e) {
-                throw new MilvusClientException(ErrorCode.INVALID_PARAMS, e.getMessage() + e.getCause().getMessage());
-            }
-        }
-
         if (request.getGroupByFieldName() != null && !request.getGroupByFieldName().isEmpty()) {
             builder.addSearchParams(
                     KeyValuePair.newBuilder()
@@ -388,6 +382,14 @@ public class VectorUtils {
         builder.setPlaceholderGroup(byteStr);
         builder.setNq(vectors.size());
 
+        // search parameters
+        // tries to fit the compatibility between v2.5.1 and older versions
+        Map<String, Object> paramMap = new HashMap<>();
+        if (null != annSearchReq.getParams() && !annSearchReq.getParams().isEmpty()) {
+            paramMap = JsonUtils.fromJson(annSearchReq.getParams(), new TypeToken<Map<String, Object>>() {}.getType());
+        }
+        ParamUtils.compatibleSearchParams(paramMap, builder);
+
         builder.addSearchParams(
                         KeyValuePair.newBuilder()
                                 .setKey(Constant.VECTOR_FIELD)
@@ -406,16 +408,6 @@ public class VectorUtils {
                             .build());
         }
 
-        // params
-        String params = "{}";
-        if (null != annSearchReq.getParams() && !annSearchReq.getParams().isEmpty()) {
-            params = annSearchReq.getParams();
-        }
-        builder.addSearchParams(KeyValuePair.newBuilder()
-                        .setKey(Constant.PARAMS)
-                        .setValue(params)
-                        .build());
-
         // always use expression since dsl is discarded
         builder.setDslType(DslType.BoolExprV1);
         if (annSearchReq.getExpr() != null && !annSearchReq.getExpr().isEmpty()) {

sdk-core/src/main/milvus-proto

@@ -1 +1 @@
-Subproject commit c18fc42493ca2768a81bf2346340325db1e6cab9
+Subproject commit 66c87c7e94889993d4493cb77c6354dac8c1047e