Browse Source

stage fix:keep the original directory structure (#1483)

Signed-off-by: lentitude2tk <xushuang.hu@zilliz.com>
xushuang.hu 1 week ago
parent
commit
76d002254e

+ 15 - 4
sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/StageOperation.java

@@ -35,6 +35,8 @@ import org.slf4j.LoggerFactory;
 
 
 import java.io.File;
 import java.io.File;
 import java.io.IOException;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.Instant;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.Date;
@@ -86,7 +88,7 @@ public class StageOperation {
                 File file = new File(localFilePath);
                 File file = new File(localFilePath);
                 long fileStartTime = System.currentTimeMillis();
                 long fileStartTime = System.currentTimeMillis();
                 try {
                 try {
-                    uploadLocalFileToStage(localFilePath);
+                    uploadLocalFileToStage(localFilePath, localDirOrFilePath);
                     long bytes = processedBytes.addAndGet(file.length());
                     long bytes = processedBytes.addAndGet(file.length());
                     long elapsed = System.currentTimeMillis() - fileStartTime;
                     long elapsed = System.currentTimeMillis() - fileStartTime;
                     double percent = totalBytes == 0 ? 100.0 : (bytes * 100.0 / totalBytes);
                     double percent = totalBytes == 0 ? 100.0 : (bytes * 100.0 / totalBytes);
@@ -145,10 +147,19 @@ public class StageOperation {
         return inputPath + "/";
         return inputPath + "/";
     }
     }
 
 
-    private String uploadLocalFileToStage(String localFilePath) throws Exception {
+    private String uploadLocalFileToStage(String localFilePath, String rootPath) throws Exception {
         File file = new File(localFilePath);
         File file = new File(localFilePath);
-        String fileName = file.getName();
-        String remoteFilePath = applyStageResponse.getUploadPath() + fileName;
+        Path filePath = file.toPath().toAbsolutePath();
+        Path root = Paths.get(rootPath).toAbsolutePath();
+
+        String relativePath;
+        if (root.toFile().isFile()) {
+            relativePath = file.getName();
+        } else {
+            relativePath = root.relativize(filePath).toString().replace("\\", "/");
+        }
+
+        String remoteFilePath = applyStageResponse.getUploadPath() + relativePath;
         putObject(file, remoteFilePath);
         putObject(file, remoteFilePath);
         return remoteFilePath;
         return remoteFilePath;
     }
     }