feat: support create collection from json (#876)

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

client/src/i18n/cn/button.ts

@@ -47,6 +47,7 @@ const btnTrans = {
   editDefaultValue: '编辑默认值',
   viewData: '查看数据',
   mmapSetting: '内存映射 (MMap) 设置',
+  importFromJSON: '从 JSON 导入',
 
   // tips
   loadColTooltip: '加载 Collection',

client/src/i18n/en/button.ts

@@ -47,6 +47,7 @@ const btnTrans = {
   EditDefaultValue: 'Edit Default Value',
   viewData: 'View Data',
   mmapSetting: 'MMap Settings',
+  importFromJSON: 'Import from JSON',
 
   // tips
   loadColTooltip: 'Load Collection',

client/src/pages/databases/collections/Types.ts

@@ -17,10 +17,10 @@ export type FunctionConfig = {
 export interface CollectionCreateParam {
   collection_name: string;
   description: string;
-  autoID: boolean;
   fields: CreateField[];
   consistency_level: string;
   functions: FunctionConfig[];
+  properties: Record<string, unknown>;
 }
 
 export type AnalyzerType = 'standard' | 'english' | 'chinese';
@@ -80,8 +80,6 @@ export type FieldType = {
 export interface CreateFieldsProps {
   fields: CreateField[];
   setFields: Dispatch<SetStateAction<CreateField[]>>;
-  autoID: boolean;
-  setAutoID: (value: boolean) => void;
   onValidationChange: (isValid: boolean) => void;
 }
 

client/src/pages/dialogs/CreateCollectionDialog.tsx

@@ -6,6 +6,7 @@ import {
   useState,
   ChangeEvent,
   useEffect,
+  useRef,
 } from 'react';
 import { useTranslation } from 'react-i18next';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
@@ -13,7 +14,12 @@ import CustomInput from '@/components/customInput/CustomInput';
 import { ITextfieldConfig } from '@/components/customInput/Types';
 import { rootContext, dataContext } from '@/context';
 import { useFormValidation } from '@/hooks';
-import { formatForm, getAnalyzerParams, TypeEnum } from '@/utils';
+import {
+  formatForm,
+  getAnalyzerParams,
+  TypeEnum,
+  parseCollectionJson,
+} from '@/utils';
 import {
   DataTypeEnum,
   ConsistencyLevelEnum,
@@ -51,14 +57,12 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
   const [form, setForm] = useState<{
     collection_name: string;
     description: string;
-    autoID: boolean;
     enableDynamicField: boolean;
     loadAfterCreate: boolean;
     functions: BM25Function[];
   }>({
     collection_name: '',
     description: '',
-    autoID: true,
     enableDynamicField: false,
     loadAfterCreate: true,
     functions: [],
@@ -73,6 +77,7 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
 
   const [consistencyLevel, setConsistencyLevel] =
     useState<ConsistencyLevelEnum>(ConsistencyLevelEnum.Bounded); // Bounded is the default value of consistency level
+  const [properties, setProperties] = useState({});
 
   const [fields, setFields] = useState<CreateField[]>([
     {
@@ -99,14 +104,8 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
     return formatForm({ collection_name });
   }, [form]);
 
-  const { validation, checkIsValid, disabled } = useFormValidation(checkedForm);
-
-  const changeIsAutoID = (value: boolean) => {
-    setForm({
-      ...form,
-      autoID: value,
-    });
-  };
+  const { validation, checkIsValid, disabled, setDisabled } =
+    useFormValidation(checkedForm);
 
   const updateCheckBox = (
     event: ChangeEvent<any>,
@@ -220,8 +219,6 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
           data.analyzer_params = getAnalyzerParams(data.analyzer_params);
         }
 
-        data.is_primary_key && (data.autoID = form.autoID);
-
         // delete sparse vector dime
         if (data.data_type === DataTypeEnum.SparseFloatVector) {
           delete data.dim;
@@ -242,6 +239,9 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
       }),
       functions: form.functions || [],
       consistency_level: consistencyLevel,
+      properties: {
+        ...properties,
+      },
     };
 
     // create collection
@@ -364,13 +364,83 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
     });
   }, [fields]);
 
+  // Import from json
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleImportClick = () => {
+    fileInputRef.current?.click();
+  };
+
+  const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+    if (!file) return;
+    const reader = new FileReader();
+    reader.onload = evt => {
+      try {
+        const json = JSON.parse(evt.target?.result as string);
+
+        if (
+          !json.collection_name ||
+          !Array.isArray(json.schema?.fields) ||
+          json.schema.fields.length === 0
+        ) {
+          openSnackBar('Invalid JSON file', 'error');
+          return;
+        }
+
+        const {
+          form: importedForm,
+          fields: importedFields,
+          consistencyLevel: importedConsistencyLevel,
+          properties: importedProperties,
+        } = parseCollectionJson(json);
+
+        setFields(importedFields);
+        setConsistencyLevel(importedConsistencyLevel);
+        setForm(importedForm);
+        setProperties(importedProperties);
+
+        // enable submit
+        setDisabled(false);
+
+        openSnackBar('Import successful', 'success');
+      } catch (err) {
+        openSnackBar('Invalid JSON file', 'error');
+      }
+    };
+    reader.readAsText(file);
+  };
+
   return (
     <DialogTemplate
+      dialogClass="create-collection-dialog"
       title={collectionTrans('createTitle', { name: form.collection_name })}
       handleClose={() => {
         handleCloseDialog();
       }}
-      leftActions={<>Import from JSON</>}
+      leftActions={
+        <>
+          <button
+            type="button"
+            onClick={handleImportClick}
+            style={{
+              cursor: 'pointer',
+              background: 'none',
+              border: 'none',
+              color: '#1976d2',
+            }}
+          >
+            {btnTrans('importFromJSON')}
+          </button>
+          <input
+            ref={fileInputRef}
+            type="file"
+            accept="application/json"
+            style={{ display: 'none' }}
+            onChange={handleFileChange}
+          />
+        </>
+      }
       confirmLabel={btnTrans('create')}
       handleConfirm={handleCreateCollection}
       confirmDisabled={disabled || !fieldsValidation}
@@ -399,8 +469,6 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
           <CreateFields
             fields={fields}
             setFields={setFields}
-            autoID={form.autoID}
-            setAutoID={changeIsAutoID}
             onValidationChange={setFieldsValidation}
           />
         </Box>

client/src/pages/dialogs/create/CreateFields.tsx

@@ -17,8 +17,6 @@ import VectorFieldRow from './rows/VectorFieldRow';
 const CreateFields: FC<CreateFieldsProps> = ({
   fields,
   setFields,
-  setAutoID,
-  autoID,
   onValidationChange,
 }) => {
   // i18n
@@ -212,7 +210,6 @@ const CreateFields: FC<CreateFieldsProps> = ({
 
   const generateRequiredFieldRow = (
     field: FieldType,
-    autoID: boolean,
     index: number,
     fields: FieldType[],
     requiredFields: FieldType[]
@@ -223,9 +220,7 @@ const CreateFields: FC<CreateFieldsProps> = ({
         <PrimaryKeyFieldRow
           field={field}
           fields={fields}
-          autoID={autoID}
           onFieldChange={changeFields}
-          setAutoID={setAutoID}
         />
       );
     }
@@ -274,13 +269,7 @@ const CreateFields: FC<CreateFieldsProps> = ({
 
       {requiredFields.map((field, index) => (
         <Fragment key={field.id}>
-          {generateRequiredFieldRow(
-            field,
-            autoID,
-            index,
-            fields,
-            requiredFields
-          )}
+          {generateRequiredFieldRow(field, index, fields, requiredFields)}
         </Fragment>
       ))}
 

client/src/pages/dialogs/create/rows/PrimaryKeyFieldRow.tsx

@@ -13,21 +13,17 @@ import { rowStyles } from './styles';
 interface PrimaryKeyFieldRowProps {
   field: FieldType;
   fields: FieldType[];
-  autoID: boolean;
   onFieldChange: (
     id: string,
     changes: Partial<FieldType>,
     isValid?: boolean
   ) => void;
-  setAutoID: (value: boolean) => void;
 }
 
 const PrimaryKeyFieldRow: FC<PrimaryKeyFieldRowProps> = ({
   field,
   fields,
-  autoID,
   onFieldChange,
-  setAutoID,
 }) => {
   const { t: collectionTrans } = useTranslation('collection');
 
@@ -50,10 +46,11 @@ const PrimaryKeyFieldRow: FC<PrimaryKeyFieldRowProps> = ({
       <PrimaryKeyTypeSelector
         value={field.data_type}
         onChange={(value: DataTypeEnum) => {
-          onFieldChange(field.id!, { data_type: value });
+          const changes: Partial<FieldType> = { data_type: value };
           if (value === DataTypeEnum.VarChar) {
-            setAutoID(false);
+            changes.autoID = false;
           }
+          onFieldChange(field.id!, changes);
         }}
       />
 
@@ -75,12 +72,11 @@ const PrimaryKeyFieldRow: FC<PrimaryKeyFieldRowProps> = ({
       <FormControlLabel
         control={
           <Switch
-            checked={autoID}
-            disabled={isVarChar}
+            checked={!!field.autoID}
             size="small"
+            disabled={isVarChar}
             onChange={() => {
-              onFieldChange(field.id!, { autoID: !autoID });
-              setAutoID(!autoID);
+              onFieldChange(field.id!, { autoID: !field.autoID });
             }}
           />
         }

client/src/utils/Format.ts

@@ -70,41 +70,6 @@ export const parseLocationSearch = (search: string) => {
   return obj;
 };
 
-export const getEnumKeyByValue = (enumObj: any, enumValue: any) => {
-  const match = Object.entries(enumObj).find(
-    ([, value]) => value === enumValue
-  );
-
-  if (match) {
-    const [key] = match;
-    return key;
-  }
-
-  return '--';
-};
-
-/**
- * @param pairs e.g. [{key: 'key', value: 'value'}]
- * @returns object, e.g. {key: value}
- */
-export const getObjFromKeyValuePair = (
-  pairs: { key: string; value: any }[]
-): { [key in string]: any } => {
-  const obj = pairs.reduce(
-    (acc, cur) => {
-      acc[cur.key] = cur.value;
-      return acc;
-    },
-    {} as { [key in string]: any }
-  );
-  return obj;
-};
-
-// BinarySubstructure includes Superstructure and Substructure
-export const checkIsBinarySubstructure = (metricLabel: string): boolean => {
-  return metricLabel === 'Superstructure' || metricLabel === 'Substructure';
-};
-
 export const isVectorType = (field: FieldObject): boolean => {
   return VectorTypes.includes(field.dataType as any);
 };
@@ -132,18 +97,6 @@ export const formatAddress = (address: string): string => {
   return ip.includes(':') ? ip : `${ip}:${DEFAULT_MILVUS_PORT}`;
 };
 
-// format the prometheus address
-export const formatPrometheusAddress = (address: string): string => {
-  let formatAddress = address;
-  // add protocal (default http)
-  const withProtocol = address.includes('http');
-  if (!withProtocol) formatAddress = 'http://' + formatAddress;
-  // add port (default 9090)
-  const withPort = address.includes(':');
-  if (!withPort) formatAddress = formatAddress + ':' + DEFAULT_PROMETHEUS_PORT;
-  return formatAddress;
-};
-
 export const formatByteSize = (
   size: number,
   capacityTrans: { [key in string]: string }
@@ -362,3 +315,91 @@ export const getAnalyzerParams = (
 
   return analyzerParams;
 };
+
+/**
+ * Parse the imported collection JSON and return an object suitable for setForm/setFields/setConsistencyLevel
+ */
+export const parseCollectionJson = (json: any) => {
+  // Parse fields
+  const fields = (json.schema?.fields || []).map((f: any, idx: number) => {
+    // Handle type_params
+    let dim = undefined;
+    let max_length = undefined;
+    let enable_analyzer = undefined;
+    let analyzer_params = undefined;
+    if (Array.isArray(f.type_params)) {
+      f.type_params.forEach((param: any) => {
+        if (param.key === 'dim') dim = Number(param.value);
+        if (param.key === 'max_length') max_length = Number(param.value);
+        if (param.key === 'enable_analyzer')
+          enable_analyzer = param.value === 'true';
+
+        if (param.key === 'analyzer_params') {
+          try {
+            analyzer_params = JSON.parse(param.value);
+          } catch (e) {
+            console.error('Failed to parse analyzer_params:', e);
+          }
+        }
+      });
+    }
+
+    return {
+      data_type: f.dataType,
+      element_type: f.elementType,
+      is_primary_key: !!f.is_primary_key,
+      name: f.name,
+      description: f.description || '',
+      isDefault: false,
+      id: String(idx + 1),
+      dim: dim ?? (f.dim ? Number(f.dim) : undefined),
+      max_length:
+        max_length ?? (f.max_length ? Number(f.max_length) : undefined),
+      enable_analyzer: enable_analyzer ?? f.enable_analyzer === 'true',
+      analyzer_params: analyzer_params,
+      is_partition_key: !!f.is_partition_key,
+      is_function_output: !!f.is_function_output,
+      autoID: !!f.autoID,
+      default_value: f.default_value || undefined,
+      enable_match: f.enable_match,
+      max_capacity: f.max_capacity,
+      nullable: f.nullable,
+      'mmap.enabled': f['mmap.enabled'],
+    };
+  });
+
+  // Parse functions
+  const functions = (json.schema?.functions || []).map((fn: any) => ({
+    name: fn.name,
+    description: fn.description,
+    type: fn.type === 'BM25' ? 1 : 0,
+    input_field_names: fn.input_field_names,
+    output_field_names: fn.output_field_names,
+    params: fn.params || {},
+  }));
+
+  // Parse properties
+  const properties: Record<string, any> = {};
+  if (json.properties) {
+    json.properties.forEach((property: any) => {
+      if (property.key && property.value) {
+        properties[property.key] = property.value;
+      }
+    });
+  }
+
+  // Parse form
+  const form = {
+    collection_name: json.collection_name || json.schema?.name || '',
+    description: json.schema?.description || '',
+    autoID: !!json.schema?.autoID,
+    enableDynamicField: !!json.schema?.enable_dynamic_field,
+    loadAfterCreate: true,
+    functions,
+  };
+
+  // Parse consistencyLevel
+  const consistencyLevel = json.consistency_level || 'Bounded';
+
+  return { form, fields, consistencyLevel, properties };
+};