Ver Fonte

Fix a bug: incorrect outputFields of query/search with milvus v2.5.8 (#1355)

Signed-off-by: yhmo <yihua.mo@zilliz.com>
groot há 1 mês atrás
pai
commit
28ca9ba54a

+ 3 - 32
examples/src/main/java/io/milvus/v1/BulkWriterExample.java

@@ -79,6 +79,7 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 
@@ -647,38 +648,8 @@ public class BulkWriterExample {
         List<QueryResultsWrapper.RowRecord> rowRecords = query(expr, Lists.newArrayList("*"));
         System.out.println("Query results:");
         for (QueryResultsWrapper.RowRecord record : rowRecords) {
-            JsonObject rowObject = new JsonObject();
-            // scalar field
-            rowObject.addProperty("id", (Long)record.get("id"));
-            rowObject.addProperty("bool", (Boolean) record.get("bool"));
-            rowObject.addProperty("int8", (Integer) record.get("int8"));
-            rowObject.addProperty("int16", (Integer) record.get("int16"));
-            rowObject.addProperty("int32", (Integer) record.get("int32"));
-            rowObject.addProperty("float", (Float) record.get("float"));
-            rowObject.addProperty("double", (Double) record.get("double"));
-            rowObject.addProperty("varchar", (String) record.get("varchar"));
-            rowObject.add("json", (JsonElement) record.get("json"));
-
-            // vector field
-            rowObject.add("float_vector", GSON_INSTANCE.toJsonTree(record.get("float_vector")));
-            rowObject.add("binary_vector", GSON_INSTANCE.toJsonTree(((ByteBuffer)record.get("binary_vector")).array()));
-            rowObject.add("float16_vector", GSON_INSTANCE.toJsonTree(((ByteBuffer)record.get("float16_vector")).array()));
-            rowObject.add("sparse_vector", GSON_INSTANCE.toJsonTree(record.get("sparse_vector")));
-
-            // array field
-            rowObject.add("array_bool", GSON_INSTANCE.toJsonTree(record.get("array_bool")));
-            rowObject.add("array_int8", GSON_INSTANCE.toJsonTree(record.get("array_int8")));
-            rowObject.add("array_int16", GSON_INSTANCE.toJsonTree(record.get("array_int16")));
-            rowObject.add("array_int32", GSON_INSTANCE.toJsonTree(record.get("array_int32")));
-            rowObject.add("array_int64", GSON_INSTANCE.toJsonTree(record.get("array_int64")));
-            rowObject.add("array_varchar", GSON_INSTANCE.toJsonTree(record.get("array_varchar")));
-            rowObject.add("array_float", GSON_INSTANCE.toJsonTree(record.get("array_float")));
-            rowObject.add("array_double", GSON_INSTANCE.toJsonTree(record.get("array_double")));
-
-            // dynamic field
-            rowObject.addProperty("dynamic", (String) record.get("dynamic"));
-
-            System.out.println(rowObject);
+            Map<String, Object> fieldValues = record.getFieldValues();
+            System.out.println(fieldValues);
         }
     }
 

+ 11 - 4
examples/src/main/java/io/milvus/v1/JsonFieldExample.java

@@ -48,8 +48,7 @@ public class JsonFieldExample {
         R<QueryResults> queryRet = client.query(QueryParam.newBuilder()
                 .withCollectionName(COLLECTION_NAME)
                 .withExpr(expr)
-                .addOutField(ID_FIELD)
-                .addOutField(JSON_FIELD)
+                .withOutFields(Arrays.asList(ID_FIELD, JSON_FIELD, "dynamic1", "dynamic2"))
                 .build());
         QueryResultsWrapper queryWrapper = new QueryResultsWrapper(queryRet.getData());
         System.out.println("\nQuery with expression: " + expr);
@@ -87,7 +86,7 @@ public class JsonFieldExample {
         );
 
         CollectionSchemaParam collectionSchemaParam = CollectionSchemaParam.newBuilder()
-                .withEnableDynamicField(false)
+                .withEnableDynamicField(true)
                 .withFieldTypes(fieldsSchema)
                 .build();
 
@@ -140,7 +139,14 @@ public class JsonFieldExample {
             }
             metadata.add("flags", gson.toJsonTree(Arrays.asList(i, i + 1, i + 2)));
             row.add(JSON_FIELD, metadata);
-            System.out.println(metadata);
+//            System.out.println(metadata);
+
+            // dynamic fields
+            if (i%2 == 0) {
+                row.addProperty("dynamic1", (double)i/3);
+            } else {
+                row.addProperty("dynamic2", "ok");
+            }
 
             client.insert(InsertParam.newBuilder()
                     .withCollectionName(COLLECTION_NAME)
@@ -166,5 +172,6 @@ public class JsonFieldExample {
         queryWithExpr(client, "JSON_CONTAINS(metadata[\"flags\"], 9)");
         queryWithExpr(client, "JSON_CONTAINS_ANY(metadata[\"flags\"], [8, 9, 10])");
         queryWithExpr(client, "JSON_CONTAINS_ALL(metadata[\"flags\"], [8, 9, 10])");
+        queryWithExpr(client, "dynamic1 < 2.0");
     }
 }

+ 2 - 0
examples/src/main/java/io/milvus/v2/BulkWriterExample.java

@@ -786,6 +786,8 @@ public class BulkWriterExample {
             comparePrint(collectionSchema, originalEntity, fetchedEntity, "binary_vector");
             comparePrint(collectionSchema, originalEntity, fetchedEntity, "float16_vector");
             comparePrint(collectionSchema, originalEntity, fetchedEntity, "sparse_vector");
+
+            System.out.println(fetchedEntity);
         }
         System.out.println("Result is correct!");
     }

+ 11 - 2
examples/src/main/java/io/milvus/v2/JsonFieldExample.java

@@ -47,7 +47,7 @@ public class JsonFieldExample {
         QueryResp queryRet = client.query(QueryReq.builder()
                 .collectionName(COLLECTION_NAME)
                 .filter(expr)
-                .outputFields(Arrays.asList(ID_FIELD, JSON_FIELD))
+                .outputFields(Arrays.asList(ID_FIELD, JSON_FIELD, "dynamic1", "dynamic2"))
                 .build());
         System.out.println("\nQuery with expression: " + expr);
         List<QueryResp.QueryResult> records = queryRet.getQueryResults();
@@ -70,6 +70,7 @@ public class JsonFieldExample {
 
         // Create collection
         CreateCollectionReq.CollectionSchema collectionSchema = CreateCollectionReq.CollectionSchema.builder()
+                .enableDynamicField(true)
                 .build();
         collectionSchema.addField(AddFieldReq.builder()
                 .fieldName(ID_FIELD)
@@ -119,7 +120,14 @@ public class JsonFieldExample {
             }
             metadata.add("flags", gson.toJsonTree(Arrays.asList(i, i + 1, i + 2)));
             row.add(JSON_FIELD, metadata);
-            System.out.println(metadata);
+//            System.out.println(metadata);
+
+            // dynamic fields
+            if (i%2 == 0) {
+                row.addProperty("dynamic1", (double)i/3);
+            } else {
+                row.addProperty("dynamic2", "ok");
+            }
 
             client.insert(InsertReq.builder()
                     .collectionName(COLLECTION_NAME)
@@ -143,6 +151,7 @@ public class JsonFieldExample {
         queryWithExpr(client, "JSON_CONTAINS(metadata[\"flags\"], 9)");
         queryWithExpr(client, "JSON_CONTAINS_ANY(metadata[\"flags\"], [8, 9, 10])");
         queryWithExpr(client, "JSON_CONTAINS_ALL(metadata[\"flags\"], [8, 9, 10])");
+        queryWithExpr(client, "dynamic1 < 2.0");
 
         client.close();
     }

+ 4 - 0
examples/src/main/java/io/milvus/v2/TextMatchExample.java

@@ -11,6 +11,7 @@ import io.milvus.v2.common.IndexParam;
 import io.milvus.v2.service.collection.request.AddFieldReq;
 import io.milvus.v2.service.collection.request.CreateCollectionReq;
 import io.milvus.v2.service.collection.request.DropCollectionReq;
+import io.milvus.v2.service.utility.request.FlushReq;
 import io.milvus.v2.service.vector.request.InsertReq;
 import io.milvus.v2.service.vector.request.QueryReq;
 import io.milvus.v2.service.vector.request.SearchReq;
@@ -145,6 +146,9 @@ public class TextMatchExample {
                 .build());
         System.out.printf("%d rows in collection\n", (long)countR.getQueryResults().get(0).getEntity().get("count(*)"));
 
+        // TEXT_MATCH requires the data is persisted
+        client.flush(FlushReq.builder().collectionNames(Collections.singletonList(COLLECTION_NAME)).build());
+
         // Query by keyword filtering expression
         queryWithFilter(client, "TEXT_MATCH(text, \"distance\")");
         queryWithFilter(client, "TEXT_MATCH(text, \"Milvus\") or TEXT_MATCH(text, \"distance\")");

+ 63 - 32
sdk-core/src/main/java/io/milvus/response/basic/RowRecordWrapper.java

@@ -32,6 +32,8 @@ import java.util.concurrent.ConcurrentHashMap;
 public abstract class RowRecordWrapper {
     // a cache for output fields
     private ConcurrentHashMap<String, FieldDataWrapper> outputFieldsData = new ConcurrentHashMap<>();
+    // a cache for output dynamic field names
+    private List<String> dynamicFieldNames = null;
 
     public abstract List<QueryResultsWrapper.RowRecord> getRowRecords();
 
@@ -69,46 +71,75 @@ public abstract class RowRecordWrapper {
      * @return <code>RowRecord</code> a row record of the result
      */
     protected QueryResultsWrapper.RowRecord buildRowRecord(QueryResultsWrapper.RowRecord record, long index) {
-        for (String outputKey : getOutputFields()) {
-            boolean isField = false;
-            for (FieldData field : getFieldDataList()) {
-                if (outputKey.equals(field.getFieldName())) {
-                    FieldDataWrapper wrapper = getFieldWrapperInternal(field);
-                    if (index < 0 || index >= wrapper.getRowCount()) {
-                        throw new ParamException("Index out of range");
-                    }
-                    Object value = wrapper.valueByIdx((int)index);
-                    if (wrapper.isJsonField()) {
-                        JsonElement jsonField = FieldDataWrapper.ParseJSONObject(value);
-                        if (wrapper.isDynamicField() && jsonField instanceof JsonObject) {
-                            JsonObject jsonObj = (JsonObject) jsonField;
-                            for (String key: jsonObj.keySet()) {
-                                record.put(key, FieldDataWrapper.ValueOfJSONElement(jsonObj.get(key)));
-                            }
-                        } else {
-                            record.put(field.getFieldName(), jsonField);
-                        }
-                    } else {
-                        record.put(field.getFieldName(), value);
-                    }
-                    isField = true;
-                    break;
-                }
+        List<String> dynamicFields = getDynamicFieldNames();
+        List<FieldData> fieldsData = getFieldDataList();
+        for (FieldData field : fieldsData) {
+            FieldDataWrapper wrapper = getFieldWrapperInternal(field);
+            if (index < 0 || index >= wrapper.getRowCount()) {
+                throw new ParamException("Index out of range");
             }
+            Object value = wrapper.valueByIdx((int)index);
+            if (wrapper.isJsonField()) {
+                JsonElement jsonValue = FieldDataWrapper.ParseJSONObject(value);
+                if (!field.getIsDynamic()) {
+                    record.put(field.getFieldName(), jsonValue);
+                    continue;
+                }
 
-            // if the output field is not a field name, fetch it from dynamic field
-            if (!isField) {
-                FieldDataWrapper dynamicField = getDynamicWrapper();
-                Object obj = dynamicField.get((int)index, outputKey);
-                if (obj != null) {
-                    record.put(outputKey, obj);
+                // dynamic field, the value must be a dict
+                if (!(jsonValue instanceof JsonObject)) {
+                    throw new ParamException("The content of dynamic field is not a JSON dict");
                 }
+
+                JsonObject jsonDict = (JsonObject)jsonValue;
+                // the outputFields of QueryRequest/SearchRequest contains a "$meta"
+                // put all key/value pairs of "$meta" into record
+                // else pick some key/value pairs according to the dynamicFields
+                for (String key: jsonDict.keySet()) {
+                    if (dynamicFields.isEmpty() || dynamicFields.contains(key)) {
+                        record.put(key, FieldDataWrapper.ValueOfJSONElement(jsonDict.get(key)));
+                    }
+                }
+            } else {
+                record.put(field.getFieldName(), value);
             }
         }
+
         return record;
     }
 
+    private List<String> getDynamicFieldNames() {
+        if (dynamicFieldNames != null) {
+            return dynamicFieldNames;
+        }
+
+        dynamicFieldNames = new ArrayList<>();
+        // find out dynamic field names
+        List<FieldData> fieldsData = getFieldDataList();
+        String dynamicFieldName = null;
+        List<String> fieldNames = new ArrayList<>();
+        for (FieldData field : fieldsData) {
+            if (!fieldNames.contains(field.getFieldName())) {
+                fieldNames.add(field.getFieldName());
+            }
+            if (field.getIsDynamic()) {
+                dynamicFieldName = field.getFieldName();
+            }
+        }
+
+        List<String> outputNames = getOutputFields();
+        for (String name : outputNames) {
+            if (name.equals(dynamicFieldName)) {
+                dynamicFieldNames.clear();
+                break;
+            }
+            if (!fieldNames.contains(name)) {
+                dynamicFieldNames.add(name);
+            }
+        }
+        return dynamicFieldNames;
+    }
+
     protected abstract List<FieldData> getFieldDataList();
     protected abstract List<String> getOutputFields();
-
 }

+ 1 - 1
sdk-core/src/test/java/io/milvus/client/MilvusClientDockerTest.java

@@ -78,7 +78,7 @@ class MilvusClientDockerTest {
     private static final TestUtils utils = new TestUtils(DIMENSION);
 
     @Container
-    private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.7");
+    private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.8");
 
     @BeforeAll
     public static void setUp() {

+ 2 - 2
sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java

@@ -80,7 +80,7 @@ class MilvusClientV2DockerTest {
     private static final TestUtils utils = new TestUtils(DIMENSION);
 
     @Container
-    private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.7");
+    private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.8");
 
     @BeforeAll
     public static void setUp() {
@@ -1932,7 +1932,7 @@ class MilvusClientV2DockerTest {
         QueryResp queryResp = client.query(QueryReq.builder()
                 .collectionName(randomCollectionName)
                 .filter("id >= 0")
-                .outputFields(Lists.newArrayList("*"))
+                .outputFields(Arrays.asList("desc", "flag"))
                 .consistencyLevel(ConsistencyLevel.STRONG)
                 .build());
         List<QueryResp.QueryResult> queryResults = queryResp.getQueryResults();