Browse Source

Avoid an exception of LocalBulkWriter in Java 24 env (#1391)

Signed-off-by: yhmo <yihua.mo@zilliz.com>
groot 1 month ago
parent
commit
7bb6977653

+ 1 - 2
sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/common/utils/ParquetReaderUtils.java

@@ -20,7 +20,6 @@
 package io.milvus.bulkwriter.common.utils;
 
 import org.apache.avro.generic.GenericData;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.parquet.avro.AvroParquetReader;
 
@@ -31,7 +30,7 @@ public abstract class ParquetReaderUtils {
         Path path = new Path(localFilePath);
         try (org.apache.parquet.hadoop.ParquetReader<GenericData.Record> reader = AvroParquetReader
                 .<GenericData.Record>builder(path)
-                .withConf(new Configuration())
+                .withConf(ParquetUtils.getParquetConfiguration())
                 .build()) {
             GenericData.Record record;
             while ((record = reader.read()) != null) {

+ 8 - 0
sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/common/utils/ParquetUtils.java

@@ -19,6 +19,7 @@
 
 package io.milvus.bulkwriter.common.utils;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.PrimitiveType;
@@ -158,4 +159,11 @@ public class ParquetUtils {
                 break;
         }
     }
+
+    public static Configuration getParquetConfiguration() {
+        // set fs.file.impl.disable.cache to true for this issue: https://github.com/milvus-io/milvus-sdk-java/issues/1381
+        Configuration configuration = new Configuration();
+        configuration.set("fs.file.impl.disable.cache", "true");
+        return configuration;
+    }
 }

+ 1 - 1
sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/writer/ParquetFileWriter.java

@@ -53,7 +53,7 @@ public class ParquetFileWriter implements FormatFileWriter {
         int rowGroupBytes = 16 * 1024 * 1024;
 
         // declare and define the ParquetWriter.
-        Configuration configuration = new Configuration();
+        Configuration configuration = ParquetUtils.getParquetConfiguration();
         GroupWriteSupport.setSchema(messageType, configuration);
         GroupWriteSupport writeSupport = new GroupWriteSupport();
         this.writer = new ParquetWriter<>(new Path(filePath),