Add ignore_growing flag for search/query (#465) (#509)

Signed-off-by: yhmo <yihua.mo@zilliz.com>

src/main/java/io/milvus/param/Constant.java

@@ -29,6 +29,7 @@ public class Constant {
     public static final String VECTOR_DIM = "dim";
     public static final String VARCHAR_MAX_LENGTH = "max_length";
     public static final String TOP_K = "topk";
+    public static final String IGNORE_GROWING = "ignore_growing";
     public static final String INDEX_TYPE = "index_type";
     public static final String METRIC_TYPE = "metric_type";
     public static final String ROUND_DECIMAL = "round_decimal";
@@ -42,6 +43,7 @@ public class Constant {
     public static final String IMPORT_PROGRESS = "progress_percent";
     public static final String DEFAULT_INDEX_NAME = "";
     public final static String OFFSET = "offset";
+    public final static String LIMIT = "limit";
 
     // constant values for general
     public static final String TTL_SECONDS = "collection.ttl.seconds";

src/main/java/io/milvus/param/ParamUtils.java

@@ -291,6 +291,11 @@ public class ParamUtils {
                         KeyValuePair.newBuilder()
                                 .setKey(Constant.ROUND_DECIMAL)
                                 .setValue(String.valueOf(requestParam.getRoundDecimal()))
+                                .build())
+                .addSearchParams(
+                        KeyValuePair.newBuilder()
+                                .setKey(Constant.IGNORE_GROWING)
+                                .setValue(String.valueOf(requestParam.isIgnoreGrowing()))
                                 .build());
 
         if (null != requestParam.getParams() && !requestParam.getParams().isEmpty()) {
@@ -346,7 +351,7 @@ public class ParamUtils {
         long offset = requestParam.getOffset();
         if (offset > 0) {
             builder.addQueryParams(KeyValuePair.newBuilder()
-                    .setKey("offset")
+                    .setKey(Constant.OFFSET)
                     .setValue(String.valueOf(offset))
                     .build());
         }
@@ -354,11 +359,17 @@ public class ParamUtils {
         long limit = requestParam.getLimit();
         if (limit > 0) {
             builder.addQueryParams(KeyValuePair.newBuilder()
-                    .setKey("limit")
+                    .setKey(Constant.LIMIT)
                     .setValue(String.valueOf(limit))
                     .build());
         }
 
+        // ignore growing
+        builder.addQueryParams(KeyValuePair.newBuilder()
+                .setKey(Constant.IGNORE_GROWING)
+                .setValue(String.valueOf(requestParam.isIgnoreGrowing()))
+                .build());
+
         return builder.build();
     }
 

src/main/java/io/milvus/param/dml/QueryParam.java

@@ -45,6 +45,7 @@ public class QueryParam {
     private final ConsistencyLevelEnum consistencyLevel;
     private final long offset;
     private final long limit;
+    private final boolean ignoreGrowing;
 
     private QueryParam(@NonNull Builder builder) {
         this.collectionName = builder.collectionName;
@@ -57,6 +58,7 @@ public class QueryParam {
         this.gracefulTime = builder.gracefulTime;
         this.offset = builder.offset;
         this.limit = builder.limit;
+        this.ignoreGrowing = builder.ignoreGrowing;
     }
 
     public static Builder newBuilder() {
@@ -77,6 +79,7 @@ public class QueryParam {
         private ConsistencyLevelEnum consistencyLevel;
         private Long offset = 0L;
         private Long limit = 0L;
+        private Boolean ignoreGrowing = Boolean.FALSE;
 
         private Builder() {
         }
@@ -229,6 +232,18 @@ public class QueryParam {
             return this;
         }
 
+        /**
+         * Ignore the growing segments to get best query performance. Default is False.
+         * For the user case that don't require data visibility.
+         *
+         * @param ignoreGrowing <code>Boolean.TRUE</code> ignore, Boolean.FALSE is not
+         * @return <code>Builder</code>
+         */
+        public Builder withIgnoreGrowing(@NonNull Boolean ignoreGrowing) {
+            this.ignoreGrowing = ignoreGrowing;
+            return this;
+        }
+
         /**
          * Verifies parameters and creates a new {@link QueryParam} instance.
          *
@@ -273,6 +288,7 @@ public class QueryParam {
                 ", consistencyLevel='" + consistencyLevel + '\'' +
                 ", offset=" + offset +
                 ", limit=" + limit +
+                ", ignoreGrowing='" + ignoreGrowing + '\'' +
                 '}';
     }
 }

src/main/java/io/milvus/param/dml/SearchParam.java

@@ -51,6 +51,7 @@ public class SearchParam {
     private final long guaranteeTimestamp;
     private final Long gracefulTime;
     private final ConsistencyLevelEnum consistencyLevel;
+    private final boolean ignoreGrowing;
 
     private SearchParam(@NonNull Builder builder) {
         this.collectionName = builder.collectionName;
@@ -68,6 +69,7 @@ public class SearchParam {
         this.guaranteeTimestamp = builder.guaranteeTimestamp;
         this.gracefulTime = builder.gracefulTime;
         this.consistencyLevel = builder.consistencyLevel;
+        this.ignoreGrowing = builder.ignoreGrowing;
     }
 
     public static Builder newBuilder() {
@@ -93,6 +95,7 @@ public class SearchParam {
         private Long guaranteeTimestamp = Constant.GUARANTEE_EVENTUALLY_TS;
         private Long gracefulTime = 5000L;
         private ConsistencyLevelEnum consistencyLevel;
+        private Boolean ignoreGrowing = Boolean.FALSE;
 
        Builder() {
         }
@@ -293,6 +296,18 @@ public class SearchParam {
             return this;
         }
 
+        /**
+         * Ignore the growing segments to get best search performance. Default is False.
+         * For the user case that don't require data visibility.
+         *
+         * @param ignoreGrowing <code>Boolean.TRUE</code> ignore, Boolean.FALSE is not
+         * @return <code>Builder</code>
+         */
+        public Builder withIgnoreGrowing(@NonNull Boolean ignoreGrowing) {
+            this.ignoreGrowing = ignoreGrowing;
+            return this;
+        }
+
         /**
          * Verifies parameters and creates a new {@link SearchParam} instance.
          *
@@ -381,6 +396,8 @@ public class SearchParam {
                 ", nq=" + NQ +
                 ", expr='" + expr + '\'' +
                 ", params='" + params + '\'' +
+                ", consistencyLevel='" + consistencyLevel + '\'' +
+                ", ignoreGrowing='" + ignoreGrowing + '\'' +
                 '}';
     }
 }

src/test/java/io/milvus/client/MilvusServiceClientTest.java

@@ -2044,6 +2044,7 @@ class MilvusServiceClientTest {
                 .withTopK(5)
                 .withVectors(vectors2)
                 .withExpr("dummy")
+                .withIgnoreGrowing(Boolean.TRUE)
                 .build()
         );
 
@@ -2127,6 +2128,8 @@ class MilvusServiceClientTest {
         // test throw exception with illegal input
         List<String> partitions = Collections.singletonList("partition1");
         List<String> outputFields = Collections.singletonList("field1");
+
+        // empty collection name
         assertThrows(ParamException.class, () -> QueryParam.newBuilder()
                 .withCollectionName("")
                 .withPartitionNames(partitions)
@@ -2137,6 +2140,7 @@ class MilvusServiceClientTest {
                 .build()
         );
 
+        // empty expression
         assertThrows(ParamException.class, () -> QueryParam.newBuilder()
                 .withCollectionName("collection1")
                 .withPartitionNames(partitions)
@@ -2145,6 +2149,7 @@ class MilvusServiceClientTest {
                 .build()
         );
 
+        // negative travel time stamp
         assertThrows(ParamException.class, () -> QueryParam.newBuilder()
                 .withCollectionName("collection1")
                 .withPartitionNames(partitions)
@@ -2154,6 +2159,7 @@ class MilvusServiceClientTest {
                 .build()
         );
 
+        // negative guarantee time stamp
         assertThrows(ParamException.class, () -> QueryParam.newBuilder()
                 .withCollectionName("collection1")
                 .withPartitionNames(partitions)
@@ -2163,6 +2169,7 @@ class MilvusServiceClientTest {
                 .build()
         );
 
+        // negative topk
         assertThrows(ParamException.class, () -> QueryParam.newBuilder()
                 .withCollectionName("collection1")
                 .withPartitionNames(partitions)
@@ -2172,6 +2179,7 @@ class MilvusServiceClientTest {
                 .build()
         );
 
+        // negative offset
         assertThrows(ParamException.class, () -> QueryParam.newBuilder()
                 .withCollectionName("collection1")
                 .withPartitionNames(partitions)
@@ -2180,6 +2188,18 @@ class MilvusServiceClientTest {
                 .withOffset(-1L)
                 .build()
         );
+
+        // success
+        assertDoesNotThrow(() -> QueryParam.newBuilder()
+                .withCollectionName("collection1")
+                .withPartitionNames(partitions)
+                .withOutFields(outputFields)
+                .withExpr("dummy")
+                .withOffset(1L)
+                .withLimit(1L)
+                .withIgnoreGrowing(Boolean.TRUE)
+                .build()
+        );
     }
 
     @Test