Multiclient test hang issue (#1555) (#1559)

Signed-off-by: yhmo <yihua.mo@zilliz.com>

docker-compose.yml

@@ -1,95 +1,47 @@
 version: '3.5'
 
 services:
-  etcd:
-    container_name: milvus-javasdk-test-etcd
-    image: quay.io/coreos/etcd:v3.5.18
-    volumes:
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
-    command: etcd -listen-peer-urls=http://127.0.0.1:2380 -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls=http://127.0.0.1:2380 --initial-cluster default=http://127.0.0.1:2380 --data-dir /etcd
-    ports:
-      - "2379:2379"
-      - "2380:2380"
-      - "4001:4001"
-
-  minio:
-    container_name: milvus-javasdk-test-minio
-    image: minio/minio:RELEASE.2024-05-28T17-19-04Z
-    ports:
-      - "9000:9000"
-      - "9001:9001"
+  standalone:
+    container_name: milvus-javasdk-standalone-1
+    image: milvusdb/milvus:v2.5.14
+    command: [ "milvus", "run", "standalone" ]
     environment:
-      MINIO_ACCESS_KEY: minioadmin
-      MINIO_SECRET_KEY: minioadmin
+      - COMMON_STORAGETYPE=local
+      - DEPLOY_MODE=STANDALONE
+      - ETCD_USE_EMBED=true
+      - ETCD_DATA_DIR=/var/lib/milvus/etcd
     volumes:
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
-    command: minio server /minio_data --console-address ":9001"
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus-1:/var/lib/milvus
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      test: [ "CMD", "curl", "-f", "http://localhost:9091/healthz" ]
       interval: 30s
+      start_period: 90s
       timeout: 20s
       retries: 3
-
-  standalone:
-    container_name: milvus-javasdk-test-standalone
-    image: milvusdb/milvus:v2.5.14
-    command: ["milvus", "run", "standalone"]
-    environment:
-      ETCD_ENDPOINTS: etcd:2379
-      MINIO_ADDRESS: minio:9000
-    volumes:
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
     ports:
       - "19530:19530"
       - "9091:9091"
-    depends_on:
-      - "etcd"
-      - "minio"
 
-  etcdslave:
-    container_name: milvus-javasdk-test-etcd-slave
-    image: quay.io/coreos/etcd:v3.5.18
-    volumes:
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd-slave:/etcd
-    command: etcd -listen-peer-urls=http://127.0.0.1:2380 -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls=http://127.0.0.1:2380 --initial-cluster default=http://127.0.0.1:2380 --data-dir /etcd
-    ports:
-      - "2381:2379"
-      - "2382:2380"
-      - "4002:4001"
-
-  minioslave:
-    container_name: milvus-javasdk-test-minio-slave
-    image: minio/minio:RELEASE.2024-05-28T17-19-04Z
-    ports:
-      - "19000:9000"
-      - "19001:9001"
+  standaloneslave:
+    container_name: milvus-javasdk-standalone-2
+    image: milvusdb/milvus:v2.5.14
+    command: [ "milvus", "run", "standalone" ]
     environment:
-      MINIO_ACCESS_KEY: minioadmin
-      MINIO_SECRET_KEY: minioadmin
+      - COMMON_STORAGETYPE=local
+      - DEPLOY_MODE=STANDALONE
+      - ETCD_USE_EMBED=true
+      - ETCD_DATA_DIR=/var/lib/milvus/etcd
     volumes:
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio-slave:/minio_data
-    command: minio server /minio_data --console-address ":9001"
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus-2:/var/lib/milvus
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      test: [ "CMD", "curl", "-f", "http://localhost:9091/healthz" ]
       interval: 30s
+      start_period: 90s
       timeout: 20s
       retries: 3
-
-  standaloneslave:
-    container_name: milvus-javasdk-test-slave-standalone
-    image: milvusdb/milvus:v2.5.14
-    command: ["milvus", "run", "standalone"]
-    environment:
-      ETCD_ENDPOINTS: etcdslave:2379
-      MINIO_ADDRESS: minioslave:9000
-    volumes:
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus-slave:/var/lib/milvus
     ports:
       - "19531:19530"
       - "9092:9091"
-    depends_on:
-      - "etcdslave"
-      - "minioslave"
 
 networks:
   default:

sdk-core/src/test/java/io/milvus/client/MilvusClientDockerTest.java

@@ -604,7 +604,7 @@ class MilvusClientDockerTest {
         for (String fieldName : outputFields) {
             FieldDataWrapper wrapper = queryResultsWrapper.getFieldWrapper(fieldName);
             System.out.println("Query data of " + fieldName + ", row count: " + wrapper.getRowCount());
-            System.out.println(wrapper.getFieldData());
+//            System.out.println(wrapper.getFieldData());
             Assertions.assertEquals(nq, wrapper.getFieldData().size());
 
             if (fieldName.compareTo("id") == 0) {
@@ -685,8 +685,8 @@ class MilvusClientDockerTest {
         SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
         for (int i = 0; i < targetVectors.size(); ++i) {
             List<SearchResultsWrapper.IDScore> scores = results.getIDScore(i);
-            System.out.println("The result of No." + i + " target vector(ID = " + targetVectorIDs.get(i) + "):");
-            System.out.println(scores);
+//            System.out.println("The result of No." + i + " target vector(ID = " + targetVectorIDs.get(i) + "):");
+//            System.out.println(scores);
             Assertions.assertEquals(targetVectorIDs.get(i), scores.get(0).getLongID());
 
             Object obj = scores.get(0).get(DataType.FloatVector.name());
@@ -965,8 +965,8 @@ class MilvusClientDockerTest {
         SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
         for (int i = 0; i < targetVectors.size(); ++i) {
             List<SearchResultsWrapper.IDScore> scores = results.getIDScore(i);
-            System.out.println("The result of No." + i + " target vector(ID = " + targetVectorIDs.get(i) + "):");
-            System.out.println(scores);
+//            System.out.println("The result of No." + i + " target vector(ID = " + targetVectorIDs.get(i) + "):");
+//            System.out.println(scores);
             Assertions.assertEquals(targetVectorIDs.get(i), scores.get(0).getLongID());
             ByteBuffer buf = (ByteBuffer) scores.get(0).get(DataType.BinaryVector.name());
             Assertions.assertArrayEquals(targetVectors.get(i).array(), buf.array());
@@ -1082,11 +1082,11 @@ class MilvusClientDockerTest {
         SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
         for (int i = 0; i < targetVectors.size(); ++i) {
             List<SearchResultsWrapper.IDScore> scores = results.getIDScore(i);
-            System.out.println("The result of No." + i + " target vector(ID = " + targetVectorIDs.get(i) + "):");
-            System.out.println(scores);
-            if (targetVectorIDs.get(i) != scores.get(0).getLongID()) {
-                System.out.println(targetVectors.get(i));
-            }
+//            System.out.println("The result of No." + i + " target vector(ID = " + targetVectorIDs.get(i) + "):");
+//            System.out.println(scores);
+//            if (targetVectorIDs.get(i) != scores.get(0).getLongID()) {
+//                System.out.println(targetVectors.get(i));
+//            }
             Assertions.assertEquals(targetVectorIDs.get(i), scores.get(0).getLongID());
 
             Object v = scores.get(0).get(DataType.SparseFloatVector.name());
@@ -1607,11 +1607,11 @@ class MilvusClientDockerTest {
 
             // verify search result
             SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
-            System.out.println("Search results:");
+//            System.out.println("Search results:");
             for (int i = 0; i < targetVectors.size(); ++i) {
                 List<SearchResultsWrapper.IDScore> scores = results.getIDScore(i);
                 Assertions.assertEquals(topK, scores.size());
-                System.out.println(scores.toString());
+//                System.out.println(scores.toString());
             }
 
             // get query results
@@ -1623,7 +1623,7 @@ class MilvusClientDockerTest {
             for (String fieldName : outputFields) {
                 FieldDataWrapper wrapper = queryResultsWrapper.getFieldWrapper(fieldName);
                 System.out.println("Query data of " + fieldName + ", row count: " + wrapper.getRowCount());
-                System.out.println(wrapper.getFieldData());
+//                System.out.println(wrapper.getFieldData());
                 Assertions.assertEquals(queryIDs.size(), wrapper.getFieldData().size());
             }
 
@@ -1807,7 +1807,7 @@ class MilvusClientDockerTest {
         for (String fieldName : outputFields) {
             FieldDataWrapper wrapper = queryResultsWrapper.getFieldWrapper(fieldName);
             System.out.println("Query data of " + fieldName + ", row count: " + wrapper.getRowCount());
-            System.out.println(wrapper.getFieldData());
+//            System.out.println(wrapper.getFieldData());
             Assertions.assertEquals(nq, wrapper.getFieldData().size());
 
             if (fieldName.compareTo("id") == 0) {
@@ -1842,8 +1842,8 @@ class MilvusClientDockerTest {
         SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
         for (int i = 0; i < targetVectors.size(); ++i) {
             List<SearchResultsWrapper.IDScore> scores = results.getIDScore(i);
-            System.out.println("The result of No." + i + " target vector(ID = " + queryIds.get(i) + "):");
-            System.out.println(scores);
+//            System.out.println("The result of No." + i + " target vector(ID = " + queryIds.get(i) + "):");
+//            System.out.println(scores);
             Assertions.assertEquals(scores.get(0).getStrID(), queryIds.get(i));
         }
 
@@ -2080,8 +2080,8 @@ class MilvusClientDockerTest {
         SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
         for (int i = 0; i < targetVectors.size(); ++i) {
             List<SearchResultsWrapper.IDScore> scores = results.getIDScore(i);
-            System.out.println("The result of No." + i + " target vector:");
-            Assertions.assertFalse(scores.isEmpty());
+//            System.out.println("The result of No." + i + " target vector:");
+//            Assertions.assertFalse(scores.isEmpty());
             SearchResultsWrapper.IDScore score = scores.get(0);
             System.out.println(score);
             Object extraMeta = score.get("dynamic");
@@ -2256,9 +2256,9 @@ class MilvusClientDockerTest {
         // verify the search result
         SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
         List<SearchResultsWrapper.IDScore> scores = results.getIDScore(0);
-        System.out.println("Search results:");
+//        System.out.println("Search results:");
         for (SearchResultsWrapper.IDScore score : scores) {
-            System.out.println(score);
+//            System.out.println(score);
             long id = score.getLongID();
             List<?> strArray = (List<?>)score.get(varcharArrayName);
             Assertions.assertEquals(id%10000, (long)strArray.size());
@@ -2286,9 +2286,9 @@ class MilvusClientDockerTest {
         Assertions.assertEquals(R.Status.Success.getCode(), searchR.getStatus().intValue());
         results = new SearchResultsWrapper(searchR.getData().getResults());
         scores = results.getIDScore(0);
-        System.out.println("Search results:");
+//        System.out.println("Search results:");
         for (SearchResultsWrapper.IDScore score : scores) {
-            System.out.println(score);
+//            System.out.println(score);
             long id = score.getLongID();
             Assertions.assertTrue(id == 10068 || id == 68 || id == 10045 || id == 45);
         }
@@ -3322,7 +3322,7 @@ class MilvusClientDockerTest {
         // verify the search result
         SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
         List<SearchResultsWrapper.IDScore> scores = results.getIDScore(0);
-        System.out.println("Search results:");
+//        System.out.println("Search results:");
         Assertions.assertEquals(10, scores.size());
         for (SearchResultsWrapper.IDScore score : scores) {
             long id = score.getLongID();
@@ -3334,7 +3334,7 @@ class MilvusClientDockerTest {
                 Assertions.assertEquals(10, fieldValues.get("flag"));
                 Assertions.assertEquals("AAA", fieldValues.get("desc"));
             }
-            System.out.println(score);
+//            System.out.println(score);
         }
     }
 

sdk-core/src/test/java/io/milvus/client/MilvusMultiClientDockerTest.java

@@ -38,6 +38,9 @@ import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Test;
 
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.concurrent.ExecutionException;
@@ -90,28 +93,36 @@ class MilvusMultiClientDockerTest {
         }
     }
 
-    private static void startDockerContainer() {
-        if (!useDockerCompose) {
-            return;
-        }
-
-        // start the test container
-        Runtime runtime = Runtime.getRuntime();
-        String bashCommand = "docker compose up -d";
-
+    private static void runCommand(String command) {
+        System.out.println(command);
         try {
-            System.out.println(bashCommand);
-            Process pro = runtime.exec(bashCommand);
-            int status = pro.waitFor();
-            if (status != 0) {
-                System.out.println("Failed to start docker compose, status " + status);
+            Runtime runtime = Runtime.getRuntime();
+            Process proc = runtime.exec(command);
+            InputStream inputStream = proc.getInputStream();
+            BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+            String line;
+            while ((line = reader.readLine()) != null) {
+                System.out.println(line);
             }
 
-            System.out.println("Milvus service started");
+            boolean ok = proc.waitFor(10, TimeUnit.MINUTES);
+            if (!ok) {
+                System.out.println("Failed to run command in time: " + command);
+            }
+            System.out.println("Command done: " + command);
         } catch (Throwable t) {
-            System.out.println("Failed to execute docker compose up: " + t.getMessage());
+            System.out.println("Failed to run command: " + command + ", error: " + t.getMessage());
+        }
+    }
+
+    private static void startDockerContainer() {
+        if (!useDockerCompose) {
+            return;
         }
 
+        runCommand("docker compose version");
+        runCommand("docker compose up -d");
+
         MultiConnectParam connectParam = multiConnectParamBuilder()
                 .withAuthorization("root", "Milvus")
                 .build();
@@ -126,37 +137,8 @@ class MilvusMultiClientDockerTest {
             return;
         }
 
-        // stop all test dockers
-        Runtime runtime = Runtime.getRuntime();
-        String bashCommand = "docker compose down";
-
-        try {
-            System.out.println("Milvus service stopping...");
-            TimeUnit.SECONDS.sleep(5);
-            System.out.println(bashCommand);
-            Process pro = runtime.exec(bashCommand);
-            int status = pro.waitFor();
-            if (status != 0) {
-                System.out.println("Failed to stop test docker containers" + pro.getOutputStream().toString());
-            }
-        } catch (Throwable t) {
-            System.out.println("Failed to execute docker compose down: " + t.getMessage());
-        }
-
-        // clean up log dir
-        runtime = Runtime.getRuntime();
-        bashCommand = "docker compose rm";
-
-        try {
-            System.out.println(bashCommand);
-            Process pro = runtime.exec(bashCommand);
-            int status = pro.waitFor();
-            if (status != 0) {
-                System.out.println("Failed to clean up test docker containers" + pro.getOutputStream().toString());
-            }
-        } catch (Throwable t) {
-            System.out.println("Failed to remove docker compose volume:" + t.getMessage());
-        }
+        runCommand("docker compose down");
+        runCommand("docker compose rm");
     }
 
     @BeforeAll
@@ -382,7 +364,7 @@ class MilvusMultiClientDockerTest {
         for (String fieldName : outputFields) {
             FieldDataWrapper wrapper = queryResultsWrapper.getFieldWrapper(fieldName);
             System.out.println("Query data of " + fieldName + ", row count: " + wrapper.getRowCount());
-            System.out.println(wrapper.getFieldData());
+//            System.out.println(wrapper.getFieldData());
             assertEquals(nq, wrapper.getFieldData().size());
 
             if (fieldName.compareTo(field1Name) == 0) {
@@ -446,8 +428,8 @@ class MilvusMultiClientDockerTest {
         SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
         for (int i = 0; i < targetVectors.size(); ++i) {
             List<SearchResultsWrapper.IDScore> scores = results.getIDScore(i);
-            System.out.println("The result of No." + i + " target vector(ID = " + targetVectorIDs.get(i) + "):");
-            System.out.println(scores);
+//            System.out.println("The result of No." + i + " target vector(ID = " + targetVectorIDs.get(i) + "):");
+//            System.out.println(scores);
             assertEquals(targetVectorIDs.get(i).longValue(), scores.get(0).getLongID());
         }
 
@@ -579,8 +561,8 @@ class MilvusMultiClientDockerTest {
         SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
         for (int i = 0; i < targetVectors.size(); ++i) {
             List<SearchResultsWrapper.IDScore> scores = results.getIDScore(i);
-            System.out.println("The result of No." + i + " target vector(ID = " + targetVectorIDs.get(i) + "):");
-            System.out.println(scores);
+//            System.out.println("The result of No." + i + " target vector(ID = " + targetVectorIDs.get(i) + "):");
+//            System.out.println(scores);
             assertEquals(targetVectorIDs.get(i).longValue(), scores.get(0).getLongID());
         }
 
@@ -720,11 +702,11 @@ class MilvusMultiClientDockerTest {
 
             // verify search result
             SearchResultsWrapper results = new SearchResultsWrapper(searchR.getData().getResults());
-            System.out.println("Search results:");
+//            System.out.println("Search results:");
             for (int i = 0; i < targetVectors.size(); ++i) {
                 List<SearchResultsWrapper.IDScore> scores = results.getIDScore(i);
                 assertEquals(topK, scores.size());
-                System.out.println(scores);
+//                System.out.println(scores);
             }
 
             // get query results
@@ -736,7 +718,7 @@ class MilvusMultiClientDockerTest {
             for (String fieldName : outputFields) {
                 FieldDataWrapper wrapper = queryResultsWrapper.getFieldWrapper(fieldName);
                 System.out.println("Query data of " + fieldName + ", row count: " + wrapper.getRowCount());
-                System.out.println(wrapper.getFieldData());
+//                System.out.println(wrapper.getFieldData());
                 assertEquals(queryIDs.size(), wrapper.getFieldData().size());
             }
 

sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java

@@ -682,7 +682,7 @@ class MilvusClientV2DockerTest {
         for (int i = 0; i < DIMENSION; ++i) {
             originVector.add((float)1/(i+1));
         }
-        System.out.println("Original float32 vector: " + originVector);
+//        System.out.println("Original float32 vector: " + originVector);
         row.add(float16Field, JsonUtils.toJsonTree(Float16Utils.f32VectorToFp16Buffer(originVector).array()));
         row.add(bfloat16Field, JsonUtils.toJsonTree(Float16Utils.f32VectorToBf16Buffer(originVector).array()));
 
@@ -716,7 +716,7 @@ class MilvusClientV2DockerTest {
             for (int i = 0; i < outputVector.size(); i++) {
                 Assertions.assertEquals(originVector.get(i), outputVector.get(i), 0.001f);
             }
-            System.out.println("Output float16 vector: " + outputVector);
+//            System.out.println("Output float16 vector: " + outputVector);
         }
 
         // search the bfloat16 vector field
@@ -742,7 +742,7 @@ class MilvusClientV2DockerTest {
             for (int i = 0; i < outputVector.size(); i++) {
                 Assertions.assertEquals(originVector.get(i), outputVector.get(i), 0.01f);
             }
-            System.out.println("Output bfloat16 vector: " + outputVector);
+//            System.out.println("Output bfloat16 vector: " + outputVector);
         }
 
         // get row count
@@ -2195,12 +2195,12 @@ class MilvusClientV2DockerTest {
         Assertions.assertEquals(1, searchResults.size());
         List<SearchResp.SearchResult> firstResults = searchResults.get(0);
         Assertions.assertEquals(10, firstResults.size());
-        System.out.println("Search results:");
+//        System.out.println("Search results:");
         for (SearchResp.SearchResult result : firstResults) {
             long id = (long)result.getId();
             Map<String, Object> entity = result.getEntity();
             checkFunc.apply(entity);
-            System.out.println(result);
+//            System.out.println(result);
         }
     }