Support duplicate collection (no copy data) (#346)

* duplicate dialog part1

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* finish duplicate

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* update i18n

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* move convert schema to utils

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* fix validation

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

---------

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

client/src/http/Collection.ts

@@ -87,6 +87,16 @@ export class Collection extends BaseModel implements CollectionData {
     });
   }
 
+  static duplicate(
+    collectionName: string,
+    params: { new_collection_name: string }
+  ) {
+    return super.create({
+      path: `${this.COLLECTIONS_URL}/${collectionName}/duplicate`,
+      data: params,
+    });
+  }
+
   static getStatistics() {
     return super.search({ path: this.COLLECTIONS_STATISTICS_URL, params: {} });
   }

client/src/i18n/cn/button.ts

@@ -24,6 +24,7 @@ const btnTrans = {
   importing: '导入中...',
   example: '生成随机向量',
   rename: '重命名',
+  duplicate: '复制',
 };
 
 export default btnTrans;

client/src/i18n/cn/collection.ts

@@ -9,7 +9,9 @@ const collectionTrans = {
   delete: '删除',
   deleteTooltip: '请至少选择一个要删除的项目。',
   rename: '重命名',
-  renameTooltip: '请选择一个要重命名的项目。',
+  renameTooltip: '请选择一个要重命名的Collection。',
+  duplicate: '复制',
+  duplicateTooltip: '请选择一个要复制的Collection。',
   newColName: '新的Collection名称',
   alias: '别名',
   aliasTooltip: '请选择一个Collection创建别名',
@@ -36,6 +38,8 @@ const collectionTrans = {
   consistencyLevelInfo:
     '一致性是指确保每个节点或副本在给定时间写入或读取数据时具有相同数据视图的属性。',
   entityCountInfo: '大约的Entity数量。',
+  duplicateCollectionInfo:
+    '复制Collection不会复制Collection中的数据。它只会使用现有的Schema创建一个新的Collection。',
 
   // create dialog
   createTitle: '创建Collection',
@@ -104,6 +108,9 @@ const collectionTrans = {
   newColNamePlaceholder: '新的Collection名称',
   newNameInfo: '只允许数字，字母和下划线。',
 
+  // duplicate dialog
+  duplicateNameExist: 'Collection已经存在。',
+
   // segment
   segments: '数据段',
   segPState: '持久数据段状态',

client/src/i18n/cn/dialog.ts

@@ -4,6 +4,7 @@ const dialogTrans = {
   deleteTitle: `删除 {{type}}`,
   renameTitle: `重命名 {{type}}`,
   releaseTitle: `发布 {{type}}`,
+  duplicateTitle: `复制 {{type}}`,
   createAlias: `为 {{type}} 创建别名`,
   compact: `压缩Collection {{type}}`,
   loadTitle: `加载 {{type}}`,

client/src/i18n/en/button.ts

@@ -24,6 +24,7 @@ const btnTrans = {
   importing: 'Importing...',
   example: 'Generate random vector',
   rename: 'Rename',
+  duplicate: 'Duplicate',
 };
 
 export default btnTrans;

client/src/i18n/en/collection.ts

@@ -9,7 +9,9 @@ const collectionTrans = {
   delete: 'delete',
   deleteTooltip: 'Please select at least one item to delete.',
   rename: 'rename',
-  renameTooltip: 'Please select one item to rename.',
+  renameTooltip: 'Please select one collection to rename.',
+  duplicate: 'duplicate',
+  duplicateTooltip: 'Please select one collection to duplicate.',
   newColName: 'New Collection Name',
   alias: 'Alias',
   aliasTooltip: 'Please select one collection to create alias',
@@ -36,6 +38,8 @@ const collectionTrans = {
   consistencyLevelInfo:
     'Consistency refers to the property that ensures every node or replica has the same view of data when writing or reading data at a given time.',
   entityCountInfo: 'Approximately entity count.',
+  duplicateCollectionInfo:
+    'Duplicating a collection does not copy the data within the collection. It only creates a new collection using the existing schema.',
 
   // create dialog
   createTitle: 'Create Collection',
@@ -107,6 +111,9 @@ const collectionTrans = {
   newColNamePlaceholder: 'New Collection Name',
   newNameInfo: 'Only numbers, letters, and underscores are allowed.',
 
+  // duplicate dialog
+  duplicateNameExist: 'A collection with this name already exists.',
+
   // segment
   segments: 'Segments',
   segPState: 'Persistent Segment State',

client/src/i18n/en/dialog.ts

@@ -4,6 +4,7 @@ const dialogTrans = {
   deleteTitle: `Drop {{type}}`,
   renameTitle: `Rename {{type}}`,
   releaseTitle: `Release {{type}}`,
+  duplicateTitle: `Duplicate {{type}}`,
   createAlias: `Create alias for {{type}}`,
   compact: `Compact collection {{type}}`,
   loadTitle: `Load {{type}}`,

client/src/pages/collections/Collections.tsx

@@ -25,6 +25,7 @@ import LoadCollectionDialog from '../dialogs/LoadCollectionDialog';
 import ReleaseCollectionDialog from '../dialogs/ReleaseCollectionDialog';
 import DropCollectionDialog from '../dialogs/DropCollectionDialog';
 import RenameCollectionDialog from '../dialogs/RenameCollectionDialog';
+import DuplicateCollectionDialog from '../dialogs/DuplicateCollectionDailog';
 import InsertDialog from '../dialogs/insert/Dialog';
 import ImportSampleDialog from '../dialogs/ImportSampleDialog';
 import { LOADING_STATE } from '@/consts';
@@ -217,12 +218,26 @@ const Collections = () => {
                     v.status === LOADING_STATE.UNLOADED ? (
                       <LoadCollectionDialog
                         collection={v.collectionName}
-                        onLoad={onLoad}
+                        onLoad={async () => {
+                          openSnackBar(
+                            successTrans('load', {
+                              name: collectionTrans('collection'),
+                            })
+                          );
+                          await fetchData();
+                        }}
                       />
                     ) : (
                       <ReleaseCollectionDialog
                         collection={v.collectionName}
-                        onRelease={onRelease}
+                        onRelease={async () => {
+                          openSnackBar(
+                            successTrans('release', {
+                              name: collectionTrans('collection'),
+                            })
+                          );
+                          await fetchData();
+                        }}
                       />
                     ),
                 },
@@ -257,70 +272,6 @@ const Collections = () => {
     orderBy,
   } = usePaginationHook(formatCollections);
 
-  const handleInsert = async (
-    collectionName: string,
-    partitionName: string,
-    fieldData: any[]
-  ): Promise<{ result: boolean; msg: string }> => {
-    const param: InsertDataParam = {
-      partition_name: partitionName,
-      fields_data: fieldData,
-    };
-    try {
-      await DataService.insertData(collectionName, param);
-      await DataService.flush(collectionName);
-      // update collections
-      fetchData();
-      return { result: true, msg: '' };
-    } catch (err: any) {
-      const {
-        response: {
-          data: { message },
-        },
-      } = err;
-      return { result: false, msg: message || '' };
-    }
-  };
-
-  const onCreate = () => {
-    openSnackBar(
-      successTrans('create', { name: collectionTrans('collection') })
-    );
-    fetchData();
-  };
-
-  const onRelease = async () => {
-    openSnackBar(
-      successTrans('release', { name: collectionTrans('collection') })
-    );
-    fetchData();
-  };
-
-  const onLoad = () => {
-    openSnackBar(successTrans('load', { name: collectionTrans('collection') }));
-    fetchData();
-  };
-
-  const onDelete = () => {
-    openSnackBar(
-      successTrans('delete', { name: collectionTrans('collection') })
-    );
-    fetchData();
-    setSelectedCollections([]);
-  };
-
-  const onRename = () => {
-    openSnackBar(
-      successTrans('rename', { name: collectionTrans('collection') })
-    );
-    fetchData();
-    setSelectedCollections([]);
-  };
-
-  const handleSearch = (value: string) => {
-    setSearch(value);
-  };
-
   const toolbarConfigs: ToolBarConfig[] = [
     {
       label: collectionTrans('create'),
@@ -329,7 +280,18 @@ const Collections = () => {
           open: true,
           type: 'custom',
           params: {
-            component: <CreateCollectionDialog onCreate={onCreate} />,
+            component: (
+              <CreateCollectionDialog
+                onCreate={async () => {
+                  openSnackBar(
+                    successTrans('create', {
+                      name: collectionTrans('collection'),
+                    })
+                  );
+                  await fetchData();
+                }}
+              />
+            ),
           },
         });
       },
@@ -354,7 +316,30 @@ const Collections = () => {
                 }
                 // user can't select partition on collection page, so default value is ''
                 defaultSelectedPartition={''}
-                handleInsert={handleInsert}
+                handleInsert={async (
+                  collectionName: string,
+                  partitionName: string,
+                  fieldData: any[]
+                ): Promise<{ result: boolean; msg: string }> => {
+                  const param: InsertDataParam = {
+                    partition_name: partitionName,
+                    fields_data: fieldData,
+                  };
+                  try {
+                    await DataService.insertData(collectionName, param);
+                    await DataService.flush(collectionName);
+                    // update collections
+                    fetchData();
+                    return { result: true, msg: '' };
+                  } catch (err: any) {
+                    const {
+                      response: {
+                        data: { message },
+                      },
+                    } = err;
+                    return { result: false, msg: message || '' };
+                  }
+                }}
               />
             ),
           },
@@ -378,8 +363,42 @@ const Collections = () => {
           params: {
             component: (
               <RenameCollectionDialog
-                cb={onRename}
+                cb={async () => {
+                  openSnackBar(
+                    successTrans('rename', {
+                      name: collectionTrans('collection'),
+                    })
+                  );
+                  await fetchData();
+                  setSelectedCollections([]);
+                }}
+                collectionName={selectedCollections[0].collectionName}
+              />
+            ),
+          },
+        });
+      },
+      label: collectionTrans('rename'),
+      // tooltip: collectionTrans('deleteTooltip'),
+      disabledTooltip: collectionTrans('renameTooltip'),
+      disabled: data => data.length !== 1,
+    },
+    {
+      icon: 'copy',
+      type: 'iconBtn',
+      onClick: () => {
+        setDialog({
+          open: true,
+          type: 'custom',
+          params: {
+            component: (
+              <DuplicateCollectionDialog
+                cb={async () => {
+                  setSelectedCollections([]);
+                  await fetchData();
+                }}
                 collectionName={selectedCollections[0].collectionName}
+                collections={collections}
               />
             ),
           },
@@ -399,7 +418,15 @@ const Collections = () => {
           params: {
             component: (
               <DropCollectionDialog
-                onDelete={onDelete}
+                onDelete={async () => {
+                  openSnackBar(
+                    successTrans('delete', {
+                      name: collectionTrans('collection'),
+                    })
+                  );
+                  await fetchData();
+                  setSelectedCollections([]);
+                }}
                 collections={selectedCollections}
               />
             ),
@@ -428,7 +455,7 @@ const Collections = () => {
       icon: 'search',
       searchText: search,
       onSearch: (value: string) => {
-        handleSearch(value);
+        setSearch(value);
       },
     },
   ];

client/src/pages/dialogs/DuplicateCollectionDailog.tsx

@@ -0,0 +1,115 @@
+import { FC, useContext, useMemo, useState } from 'react';
+import { Typography, makeStyles, Theme } from '@material-ui/core';
+import { useTranslation } from 'react-i18next';
+import { rootContext } from '@/context';
+import DialogTemplate from '@/components/customDialog/DialogTemplate';
+import CustomInput from '@/components/customInput/CustomInput';
+import { formatForm } from '@/utils';
+import { useFormValidation } from '@/hooks';
+import { ITextfieldConfig } from '@/components/customInput/Types';
+import { Collection } from '@/http';
+import { DuplicateCollectionDialogProps } from './Types';
+
+const useStyles = makeStyles((theme: Theme) => ({
+  wrapper: {
+    width: theme.spacing(48),
+  },
+  desc: {
+    margin: '8px 0 16px 0',
+  },
+}));
+
+const DuplicateCollectionDialog: FC<DuplicateCollectionDialogProps> = props => {
+  const { cb, collectionName, collections } = props;
+  const [form, setForm] = useState({
+    duplicate: `${collectionName}_duplicate`,
+  });
+
+  const classes = useStyles();
+
+  const checkedForm = useMemo(() => {
+    const { duplicate } = form;
+    return formatForm({ duplicate });
+  }, [form]);
+
+  const { validation, checkIsValid, disabled } = useFormValidation(checkedForm);
+
+  const { handleCloseDialog } = useContext(rootContext);
+  const { t: dialogTrans } = useTranslation('dialog');
+  const { t: warningTrans } = useTranslation('warning');
+  const { t: collectionTrans } = useTranslation('collection');
+  const { t: btnTrans } = useTranslation('btn');
+
+  const handleInputChange = (value: string) => {
+    setForm({ duplicate: value });
+  };
+
+  const handleConfirm = async () => {
+    // duplicate
+    await Collection.duplicate(collectionName, {
+      new_collection_name: form.duplicate,
+    });
+    // close dialog
+    handleCloseDialog();
+    cb && cb();
+  };
+
+  const duplicateInputConfig: ITextfieldConfig = {
+    label: collectionTrans('newColNamePlaceholder'),
+    key: 'duplicate',
+    onChange: handleInputChange,
+    variant: 'filled',
+    placeholder: collectionTrans('newColNamePlaceholder'),
+    fullWidth: true,
+    validations: [
+      {
+        rule: 'require',
+        errorText: warningTrans('required', {
+          name: collectionTrans('name'),
+        }),
+      },
+      {
+        rule: 'collectionName',
+        errorText: collectionTrans('nameContentWarning'),
+      },
+      {
+        rule: 'custom',
+        extraParam: {
+          compare: (value) => {
+            return !collections.some(collection => collection.collectionName === value);
+          },
+        },
+        errorText: collectionTrans('duplicateNameExist'),
+      },
+    ],
+    defaultValue: form.duplicate,
+  };
+
+  return (
+    <DialogTemplate
+      dialogClass={classes.wrapper}
+      title={dialogTrans('duplicateTitle', {
+        type: collectionName,
+      })}
+      handleClose={handleCloseDialog}
+      children={
+        <>
+          <Typography variant="body1" component="p" className={classes.desc}>
+            {collectionTrans('duplicateCollectionInfo')}
+          </Typography>
+          <CustomInput
+            type="text"
+            textConfig={duplicateInputConfig}
+            checkValid={checkIsValid}
+            validInfo={validation}
+          />
+        </>
+      }
+      confirmLabel={btnTrans('duplicate')}
+      handleConfirm={handleConfirm}
+      confirmDisabled={disabled}
+    />
+  );
+};
+
+export default DuplicateCollectionDialog;

client/src/pages/dialogs/Types.ts

@@ -29,6 +29,10 @@ export interface CreateAliasProps {
   cb?: () => void;
 }
 
+export interface DuplicateCollectionDialogProps extends CreateAliasProps {
+  collections: Collection[];
+}
+
 export interface RenameCollectionProps {
   collectionName: string;
   cb?: () => void;

client/src/utils/Validation.ts

@@ -17,7 +17,9 @@ export type ValidType =
   | 'multiple'
   | 'partitionName'
   | 'firstCharacter'
-  | 'specValueOrRange';
+  | 'specValueOrRange'
+  | 'duplicate'
+  | 'custom';
 export interface ICheckMapParam {
   value: string;
   extraParam?: IExtraParam;
@@ -37,6 +39,8 @@ export interface IExtraParam {
 
   // used for check start item
   invalidTypes?: TypeEnum[];
+  // used for custom validation
+  compare?: (value?: any) => boolean;
 }
 export type CheckMap = {
   [key in ValidType]: boolean;
@@ -201,6 +205,13 @@ export const checkSpecValueOrRange = (param: {
   );
 };
 
+export const checkDuplicate = (param: {
+  value: string | number;
+  compare: string | number;
+}) => {
+  return param.value !== param.compare;
+};
+
 export const getCheckResult = (param: ICheckMapParam): boolean => {
   const { value, extraParam = {}, rule } = param;
   const numberValue = Number(value);
@@ -241,6 +252,11 @@ export const getCheckResult = (param: ICheckMapParam): boolean => {
       max: extraParam?.max || 0,
       compareValue: Number(extraParam.compareValue) || 0,
     }),
+    duplicate: checkDuplicate({ value, compare: extraParam.compareValue! }),
+    custom:
+      extraParam && typeof extraParam.compare === 'function'
+        ? extraParam.compare(value)
+        : true,
   };
 
   return checkMap[rule];

server/src/collections/collections.controller.ts

@@ -10,6 +10,7 @@ import {
   VectorSearchDto,
   QueryDto,
   RenameCollectionDto,
+  DuplicateCollectionDto,
 } from './dto';
 import { LoadCollectionReq } from '@zilliz/milvus2-sdk-node';
 
@@ -49,6 +50,11 @@ export class CollectionController {
       dtoValidationMiddleware(RenameCollectionDto),
       this.renameCollection.bind(this)
     );
+    this.router.post(
+      '/:name/duplicate',
+      dtoValidationMiddleware(DuplicateCollectionDto),
+      this.duplicateCollection.bind(this)
+    );
     this.router.delete('/:name/alias/:alias', this.dropAlias.bind(this));
     // collection with index info
     this.router.get('/:name', this.describeCollection.bind(this));
@@ -143,6 +149,20 @@ export class CollectionController {
     }
   }
 
+  async duplicateCollection(req: Request, res: Response, next: NextFunction) {
+    const name = req.params?.name;
+    const data = req.body;
+    try {
+      const result = await this.collectionsService.duplicateCollection({
+        collection_name: name,
+        ...data,
+      });
+      res.send(result);
+    } catch (error) {
+      next(error);
+    }
+  }
+
   async dropCollection(req: Request, res: Response, next: NextFunction) {
     const name = req.params?.name;
     try {

server/src/collections/collections.service.ts

@@ -21,9 +21,16 @@ import {
   CompactReq,
   HasCollectionReq,
   CountReq,
+  FieldSchema,
 } from '@zilliz/milvus2-sdk-node';
 import { Parser } from '@json2csv/plainjs';
-import { throwErrorFromSDK, findKeyValue, genRows, ROW_COUNT } from '../utils';
+import {
+  throwErrorFromSDK,
+  findKeyValue,
+  genRows,
+  ROW_COUNT,
+  convertFieldSchemaToFieldType,
+} from '../utils';
 import { QueryDto, ImportSampleDto, GetReplicasDto } from './dto';
 import { CollectionData } from '../types';
 import { SchemaService } from '../schema/schema.service';
@@ -358,4 +365,27 @@ export class CollectionsService {
     throwErrorFromSDK(res.status);
     return res;
   }
+
+  async duplicateCollection(data: RenameCollectionReq) {
+    const collection: any = await this.describeCollection({
+      collection_name: data.collection_name,
+    });
+
+    const createCollectionParams: CreateCollectionReq = {
+      collection_name: data.new_collection_name,
+      fields: collection.schema.fields.map(convertFieldSchemaToFieldType),
+      consistency_level: collection.consistency_level,
+      enable_dynamic_field: !!collection.enable_dynamic_field,
+    };
+
+    if (
+      collection.schema.fields.some((f: FieldSchema) => f.is_partition_key) &&
+      collection.num_partitions
+    ) {
+      createCollectionParams.num_partitions = Number(collection.num_partitions);
+    }
+
+    console.dir(createCollectionParams, { depth: null });
+    return await this.createCollection(createCollectionParams);
+  }
 }

server/src/collections/dto.ts

@@ -123,3 +123,9 @@ export class RenameCollectionDto {
   @IsNotEmpty({ message: 'new_collection_name is empty.' })
   new_collection_name: string;
 }
+
+export class DuplicateCollectionDto {
+  @IsString()
+  @IsNotEmpty({ message: 'new_collection_name is empty.' })
+  new_collection_name: string;
+}

server/src/utils/Helper.ts

@@ -1,4 +1,9 @@
-import { KeyValuePair, FieldSchema } from '@zilliz/milvus2-sdk-node';
+import {
+  KeyValuePair,
+  FieldSchema,
+  convertToDataType,
+  FieldType,
+} from '@zilliz/milvus2-sdk-node';
 
 export const findKeyValue = (obj: KeyValuePair[], key: string) =>
   obj.find(v => v.key === key)?.value;
@@ -94,3 +99,25 @@ export const genRows = (
   size: number,
   enableDynamicField: boolean = false
 ) => Array.from({ length: size }, () => genRow(fields, enableDynamicField));
+
+export const convertFieldSchemaToFieldType = (fieldSchema: FieldSchema) => {
+  const fieldType: FieldType = {
+    name: fieldSchema.name,
+    description: fieldSchema.description,
+    data_type: convertToDataType(fieldSchema.data_type),
+    element_type: convertToDataType(fieldSchema.element_type),
+    is_primary_key: fieldSchema.is_primary_key,
+    is_partition_key: fieldSchema.is_partition_key,
+    autoID: fieldSchema.autoID,
+  };
+
+  // Convert type_params from array to object
+  if (fieldSchema.type_params) {
+    fieldType.type_params = {};
+    for (const param of fieldSchema.type_params) {
+      fieldType.type_params[param.key] = param.value;
+    }
+  }
+
+  return fieldType;
+};