Add import sample data and the refresh button and other improvments (#107)

* part1

* update load sample pos and some bugs

* update load-sample

* adjust query tab

* update text

* rename

* adjust text

client/package.json

@@ -9,7 +9,7 @@
     "@date-io/dayjs": "1.x",
     "@loadable/component": "^5.15.0",
     "@material-ui/core": "4.11.4",
-    "@material-ui/icons": "^4.11.2",
+    "@material-ui/icons": "^4.11.3",
     "@material-ui/lab": "4.0.0-alpha.58",
     "@material-ui/pickers": "^3.3.10",
     "@mui/x-data-grid": "^4.0.0",
@@ -83,4 +83,4 @@
     "@types/webpack-env": "^1.16.3",
     "prettier": "2.3.2"
   }
-}
+}

client/src/components/grid/Table.tsx

@@ -247,7 +247,7 @@ const EnhancedTable: FC<TableType> = props => {
                                 ? classes.hoverActionCell
                                 : ''
                             }`}
-                            key="manage"
+                            key={colDef.id}
                             style={cellStyle}
                           >
                             <ActionBar

client/src/components/grid/TableHead.tsx

@@ -29,7 +29,6 @@ const useStyles = makeStyles(theme => ({
     // borderBottom: 'none',
   },
   tableHeader: {
-    textTransform: 'capitalize',
     color: 'rgba(0, 0, 0, 0.6)',
     fontSize: '12.8px',
   },

client/src/components/icons/Icons.tsx

@@ -19,7 +19,7 @@ import ExitToAppIcon from '@material-ui/icons/ExitToApp';
 import ArrowForwardIosIcon from '@material-ui/icons/ArrowForwardIos';
 import RemoveCircleOutlineIcon from '@material-ui/icons/RemoveCircleOutline';
 import ArrowDropDownIcon from '@material-ui/icons/ArrowDropDown';
-import CachedIcon from '@material-ui/icons/Cached';
+import RefreshIcon from '@material-ui/icons/Refresh';
 import FilterListIcon from '@material-ui/icons/FilterList';
 import AlternateEmailIcon from '@material-ui/icons/AlternateEmail';
 import DatePicker from '@material-ui/icons/Event';
@@ -61,7 +61,7 @@ const icons: { [x in IconsType]: (props?: any) => React.ReactElement } = {
   rightArrow: (props = {}) => <ArrowForwardIosIcon {...props} />,
   remove: (props = {}) => <RemoveCircleOutlineIcon {...props} />,
   dropdown: (props = {}) => <ArrowDropDownIcon {...props} />,
-  refresh: (props = {}) => <CachedIcon {...props} />,
+  refresh: (props = {}) => <RefreshIcon {...props} />,
   filter: (props = {}) => <FilterListIcon {...props} />,
   alias: (props = {}) => <AlternateEmailIcon {...props} />,
   datePicker: (props = {}) => <DatePicker {...props} />,
@@ -114,6 +114,11 @@ const icons: { [x in IconsType]: (props?: any) => React.ReactElement } = {
   copyExpression: (props = {}) => (
     <SvgIcon viewBox="0 0 16 16" component={CopyIcon} {...props} />
   ),
+  source: (props = {}) => (
+    <SvgIcon viewBox="0 0 24 24" {...props}>
+      <path d="M20 6h-8l-2-2H4c-1.1 0-1.99.9-1.99 2L2 18c0 1.1.9 2 2 2h16c1.1 0 2-.9 2-2V8c0-1.1-.9-2-2-2zm-6 10H6v-2h8v2zm4-4H6v-2h12v2z"></path>
+    </SvgIcon>
+  ),
 };
 
 export default icons;

client/src/components/icons/Types.ts

@@ -36,4 +36,5 @@ export type IconsType =
   | 'copyExpression'
   | 'alias'
   | 'datePicker'
-  | 'download';
+  | 'download'
+  | 'source';

client/src/components/insert/ImportSample.tsx

@@ -0,0 +1,144 @@
+import { makeStyles, Theme, Typography } from '@material-ui/core';
+import { FC, useState, useContext } from 'react';
+import { useTranslation } from 'react-i18next';
+import DialogTemplate from '../customDialog/DialogTemplate';
+import CustomSelector from '../customSelector/CustomSelector';
+import { rootContext } from '../../context/Root';
+import { InsertStatusEnum } from './Types';
+
+const getStyles = makeStyles((theme: Theme) => ({
+  icon: {
+    fontSize: '16px',
+  },
+
+  selectors: {
+    '& .selectorWrapper': {
+      display: 'flex',
+      flexDirection: 'column',
+      marginBottom: theme.spacing(2),
+
+      '& .selectLabel': {
+        fontSize: '14px',
+        lineHeight: '20px',
+        color: theme.palette.attuDark.main,
+      },
+
+      '& .description': {
+        color: theme.palette.attuGrey.dark,
+        marginBottom: theme.spacing(1),
+        fontSize: 12,
+      },
+    },
+
+    '& .selector': {
+      minWidth: '128px',
+    },
+  },
+}));
+
+/**
+ * this component contains processes during insert
+ * including import, preview and status
+ */
+
+const ImportSample: FC<{ collection: string; handleImport: Function }> =
+  props => {
+    const classes = getStyles();
+    const [size, setSize] = useState<string>('100');
+    const [insertStatus, setInsertStatus] = useState<InsertStatusEnum>(
+      InsertStatusEnum.init
+    );
+
+    const { t: insertTrans } = useTranslation('insert');
+    const { t: btnTrans } = useTranslation('btn');
+    const { handleCloseDialog, openSnackBar } = useContext(rootContext);
+    // selected collection name
+
+    const sizeOptions = [
+      {
+        label: '100',
+        value: '100',
+      },
+      {
+        label: '1k',
+        value: '1000',
+      },
+      {
+        label: '10k',
+        value: '10000',
+      },
+      {
+        label: '50k',
+        value: '50000',
+      },
+    ];
+
+    const handleNext = async () => {
+      if (insertStatus === InsertStatusEnum.success) {
+        handleCloseDialog();
+        return;
+      }
+      // start loading
+      setInsertStatus(InsertStatusEnum.loading);
+      const { result, msg } = await props.handleImport(
+        props.collection,
+        size
+      );
+
+      if (!result) {
+        openSnackBar(msg, 'error');
+        setInsertStatus(InsertStatusEnum.init);
+        return;
+      }
+      setInsertStatus(InsertStatusEnum.success);
+      // hide dialog
+      handleCloseDialog();
+    };
+
+    return (
+      <DialogTemplate
+        title={insertTrans('importSampleData', { collection: props.collection })}
+        handleClose={handleCloseDialog}
+        confirmLabel={
+          insertStatus === InsertStatusEnum.init
+            ? btnTrans('import')
+            : insertStatus === InsertStatusEnum.loading
+            ? btnTrans('importing')
+            : insertStatus === InsertStatusEnum.success
+            ? btnTrans('done')
+            : insertStatus
+        }
+        handleConfirm={handleNext}
+        confirmDisabled={false}
+        showActions={true}
+        showCancel={false}
+        // don't show close icon when insert not finish
+        // showCloseIcon={insertStatus !== InsertStatusEnum.loading}
+      >
+        <form className={classes.selectors}>
+          <div className="selectorWrapper">
+            <div className="description">
+              <Typography variant="inherit" component="p">
+                {insertTrans('importSampleDataDesc')}
+              </Typography>
+            </div>
+
+            <CustomSelector
+              label={insertTrans('sampleDataSize')}
+              options={sizeOptions}
+              wrapperClass="selector"
+              labelClass="selectLabel"
+              value={size}
+              variant="filled"
+              onChange={(e: { target: { value: unknown } }) => {
+                const size = e.target.value;
+                setSize(size as string);
+              }}
+            />
+          </div>
+        </form>
+      </DialogTemplate>
+    );
+  };
+
+export default ImportSample;

client/src/http/Collection.ts

@@ -3,6 +3,7 @@ import {
   CollectionView,
   DeleteEntitiesReq,
   InsertDataParam,
+  LoadSampleParam
 } from '../pages/collections/Types';
 import { Field } from '../pages/schema/Types';
 import { VectorSearchParam } from '../types/SearchTypes';
@@ -88,6 +89,13 @@ export class CollectionHttp extends BaseModel implements CollectionView {
     });
   }
 
+  static importSample(collectionName: string, param: LoadSampleParam) {
+    return super.create({
+      path: `${this.COLLECTIONS_URL}/${collectionName}/importSample`,
+      data: param,
+    });
+  }
+
   static deleteEntities(collectionName: string, param: DeleteEntitiesReq) {
     return super.update({
       path: `${this.COLLECTIONS_URL}/${collectionName}/entities`,

client/src/i18n/cn/success.ts

@@ -2,7 +2,7 @@ const successTrans = {
   connect: 'Connection to milvus successful',
   create: `{{name}} has been created`,
   load: `{{name}} has been loaded`,
-  delete: `{{name}} successfully deleted`,
+  delete: `Delete {{name}} successfully`,
   release: `{{name}} has been released`,
 };
 

client/src/i18n/en/button.ts

@@ -12,11 +12,15 @@ const btnTrans = {
   release: 'Release',
   load: 'Load',
   insert: 'Import Data',
+  refresh: 'Refresh',
   next: 'Next',
   previous: 'Previous',
   done: 'Done',
   vectorSearch: 'Vector search',
   query: 'Query',
+  importSampleData: 'Import Sample data',
+  loading: 'Loading...',
+  importing: 'Importing...'
 };
 
 export default btnTrans;

client/src/i18n/en/insert.ts

@@ -28,6 +28,10 @@ const insertTrans = {
   statusLoadingTip: 'Please wait patiently, thank you',
   statusSuccess: 'Import Data Successfully!',
   statusError: 'Import Data Failed!',
+
+  importSampleData: 'Import sample data into {{collection}}',
+  sampleDataSize: 'Choose sample data size',
+  importSampleDataDesc: `Import random data based on the collection's schema.`
 };
 
 export default insertTrans;

client/src/pages/collections/Collection.tsx

@@ -40,10 +40,6 @@ const Collection = () => {
   };
 
   const tabs: ITab[] = [
-    {
-      label: collectionTrans('queryTab'),
-      component: <Query collectionName={collectionName} />,
-    },
     {
       label: collectionTrans('schemaTab'),
       component: <Schema collectionName={collectionName} />,
@@ -52,6 +48,10 @@ const Collection = () => {
       label: collectionTrans('partitionTab'),
       component: <Partitions collectionName={collectionName} />,
     },
+    {
+      label: collectionTrans('queryTab'),
+      component: <Query collectionName={collectionName} />,
+    },
   ];
 
   return (

client/src/pages/collections/Collections.tsx

@@ -9,6 +9,7 @@ import {
   CollectionView,
   DataTypeEnum,
   InsertDataParam,
+  LoadSampleParam,
 } from './Types';
 import { ColDefinitionsType, ToolBarConfig } from '../../components/grid/Types';
 import { usePaginationHook } from '../../hooks/Pagination';
@@ -31,6 +32,7 @@ import {
 import Highlighter from 'react-highlight-words';
 import { parseLocationSearch } from '../../utils/Format';
 import InsertContainer from '../../components/insert/Container';
+import ImportSample from '../../components/insert/ImportSample';
 import { MilvusHttp } from '../../http/Milvus';
 import { LOADING_STATE } from '../../consts/Milvus';
 import { webSokcetContext } from '../../context/WebSocket';
@@ -88,6 +90,7 @@ const Collections = () => {
   const LoadIcon = icons.load;
   const ReleaseIcon = icons.release;
   const InfoIcon = icons.info;
+  const SourceIcon = icons.source;
 
   const searchedCollections = useMemo(
     () => collections.filter(collection => collection._name.includes(search)),
@@ -183,6 +186,28 @@ const Collections = () => {
     }
   };
 
+  const handleImportSample = async (
+    collectionName: string,
+    size: string
+  ): Promise<{ result: boolean; msg: string }> => {
+    const param: LoadSampleParam = {
+      collection_name: collectionName,
+      size: size,
+    };
+    try {
+      await CollectionHttp.importSample(collectionName, param);
+      await MilvusHttp.flush(collectionName);
+      return { result: true, msg: '' };
+    } catch (err: any) {
+      const {
+        response: {
+          data: { message },
+        },
+      } = err;
+      return { result: false, msg: message || '' };
+    }
+  };
+
   const handleCreateCollection = async (param: CollectionCreateParam) => {
     const data: CollectionCreateParam = JSON.parse(JSON.stringify(param));
     const vectorType = [DataTypeEnum.BinaryVector, DataTypeEnum.FloatVector];
@@ -294,6 +319,14 @@ const Collections = () => {
         collectionList.length === 0 || selectedCollections.length > 1,
       btnVariant: 'outlined',
     },
+    {
+      type: 'iconBtn',
+      onClick: () => {
+        fetchData();
+      },
+      label: collectionTrans('delete'),
+      icon: 'refresh',
+    },
     {
       type: 'iconBtn',
       onClick: () => {
@@ -369,12 +402,6 @@ const Collections = () => {
       sortBy: '_status',
       label: collectionTrans('status'),
     },
-    {
-      id: 'consistency_level',
-      align: 'left',
-      disablePadding: false,
-      label: collectionTrans('consistencyLevel'),
-    },
     {
       id: '_rowCount',
       align: 'left',
@@ -388,6 +415,12 @@ const Collections = () => {
         </span>
       ),
     },
+    {
+      id: 'consistency_level',
+      align: 'left',
+      disablePadding: false,
+      label: collectionTrans('consistencyLevel'),
+    },
     {
       id: '_desc',
       align: 'left',
@@ -436,6 +469,37 @@ const Collections = () => {
         },
       ],
     },
+    {
+      id: 'import',
+      align: 'center',
+      disablePadding: false,
+      label: '',
+      showActionCell: true,
+      isHoverAction: true,
+      actionBarConfigs: [
+        {
+          onClick: (e: React.MouseEvent, row: CollectionView) => {
+            setDialog({
+              open: true,
+              type: 'custom',
+              params: {
+                component: (
+                  <ImportSample
+                    collection={row._name}
+                    handleImport={handleImportSample}
+                  />
+                ),
+              },
+            });
+          },
+          icon: 'source',
+          label: 'Import',
+          showIconMethod: 'renderFn',
+          getLabel: () => 'Import sample data',
+          renderIconFn: (row: CollectionView) => <SourceIcon />,
+        },
+      ],
+    },
   ];
 
   const handleSelectChange = (value: any) => {

client/src/pages/collections/Types.ts

@@ -110,6 +110,12 @@ export interface InsertDataParam {
   fields_data: any[];
 }
 
+export interface LoadSampleParam {
+  collection_name: string;
+  // e.g. [{vector: [1,2,3], age: 10}]
+  size: string;
+}
+
 export interface DeleteEntitiesReq {
   expr: string;
   partition_name?: string;

client/yarn.lock

@@ -1542,10 +1542,10 @@
     react-is "^16.8.0 || ^17.0.0"
     react-transition-group "^4.4.0"
 
-"@material-ui/icons@^4.11.2":
-  version "4.11.2"
-  resolved "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.2.tgz#b3a7353266519cd743b6461ae9fdfcb1b25eb4c5"
-  integrity sha512-fQNsKX2TxBmqIGJCSi3tGTO/gZ+eJgWmMJkgDiOfyNaunNaxcklJQFaFogYcFl0qFuaEz1qaXYXboa/bUXVSOQ==
+"@material-ui/icons@^4.11.3":
+  version "4.11.3"
+  resolved "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz#b0693709f9b161ce9ccde276a770d968484ecff1"
+  integrity sha512-IKHlyx6LDh8n19vzwH5RtHIOHl9Tu90aAAxcbWME6kp4dmvODM3UvOHJeMIDzUbd4muuJKHmlNoBN+mDY4XkBA==
   dependencies:
     "@babel/runtime" "^7.4.4"
 

server/src/__tests__/__mocks__/consts.ts

@@ -17,7 +17,7 @@ export const mockCollections = [
       fields: [
         {
           name: 'vector_field',
-          data_type: 'data_type',
+          data_type: 'FloatVector',
           type_params: [
             {
               key: 'dim',
@@ -44,7 +44,7 @@ export const mockCollections = [
       fields: [
         {
           name: 'vector_field',
-          data_type: 'data_type',
+          data_type: 'FloatVector',
           type_params: [
             {
               key: 'dim',
@@ -91,7 +91,7 @@ export const mockGetAllCollectionsData = [
       fields: [
         {
           name: 'vector_field',
-          data_type: 'data_type',
+          data_type: 'FloatVector',
           type_params: [
             {
               key: 'dim',
@@ -121,7 +121,7 @@ export const mockGetAllCollectionsData = [
       fields: [
         {
           name: 'vector_field',
-          data_type: 'data_type',
+          data_type: 'FloatVector',
           type_params: [
             {
               key: 'dim',

server/src/__tests__/collections/collections.service.test.ts

@@ -171,6 +171,24 @@ describe('Test collections service', () => {
     }
   });
 
+  test('test importSample method', async () => {
+    const mockParam = {
+      collection_name: 'c1',
+      size: 2
+    };
+    const res = await service.importSample(mockParam);
+    expect(res.data.fields_data.length).toEqual(2);
+
+    try {
+      await service.importSample({
+        collection_name: '',
+        size: 20
+      });
+    } catch (err) {
+      expect(err).toBe(ERR_NO_COLLECTION);
+    }
+  });
+
   test('test vectorSearch method', async () => {
     const mockParam = {
       collection_name: 'c1',

server/src/collections/collections.controller.ts

@@ -6,6 +6,7 @@ import {
   CreateAliasDto,
   CreateCollectionDto,
   InsertDataDto,
+  ImportSampleDto,
   VectorSearchDto,
   QueryDto,
 } from './dto';
@@ -59,6 +60,12 @@ export class CollectionController {
       this.insert.bind(this)
     );
 
+    this.router.post(
+      '/:name/importSample',
+      dtoValidationMiddleware(ImportSampleDto),
+      this.importSample.bind(this)
+    );
+
     // we need use req.body, so we can't use delete here
     this.router.put('/:name/entities', this.deleteEntities.bind(this));
 
@@ -208,6 +215,17 @@ export class CollectionController {
     }
   }
 
+  async importSample(req: Request, res: Response, next: NextFunction) {
+    const data = req.body;
+    try {
+      const result = await this.collectionsService.importSample({
+        ...data,
+      });
+      res.send(result);
+    } catch (error) {
+      next(error);
+    }
+  }
   async deleteEntities(req: Request, res: Response, next: NextFunction) {
     const name = req.params?.name;
     const data = req.body;

server/src/collections/collections.service.ts

@@ -11,7 +11,7 @@ import {
   SearchReq,
 } from '@zilliz/milvus2-sdk-node/dist/milvus/types';
 import { throwErrorFromSDK } from '../utils/Error';
-import { findKeyValue } from '../utils/Helper';
+import { findKeyValue, genRows } from '../utils/Helper';
 import { ROW_COUNT } from '../utils/Const';
 import {
   AlterAliasReq,
@@ -20,7 +20,7 @@ import {
   ShowCollectionsReq,
   ShowCollectionsType,
 } from '@zilliz/milvus2-sdk-node/dist/milvus/types/Collection';
-import { QueryDto } from './dto';
+import { QueryDto, ImportSampleDto } from './dto';
 import { DeleteEntitiesReq } from '@zilliz/milvus2-sdk-node/dist/milvus/types/Data';
 
 export class CollectionsService {
@@ -258,4 +258,17 @@ export class CollectionsService {
     }
     return data;
   }
+
+  /**
+   * Load sample data into collection
+   */
+  async importSample({ collection_name, size }: ImportSampleDto) {
+    const collectionInfo = await this.describeCollection({ collection_name });
+    const fields_data = genRows(
+      collectionInfo.schema.fields,
+      parseInt(size, 10)
+    );
+
+    return await this.insert({ collection_name, fields_data });
+  }
 }

server/src/collections/dto.ts

@@ -8,13 +8,13 @@ import {
   IsEnum,
   ArrayMinSize,
   IsObject,
-} from "class-validator";
+} from 'class-validator';
 import {
   FieldType,
   ShowCollectionsType,
-} from "@zilliz/milvus2-sdk-node/dist/milvus/types/Collection";
-import { DataType } from "@zilliz/milvus2-sdk-node/dist/milvus/types/Common";
-import { SearchParam } from "@zilliz/milvus2-sdk-node/dist/milvus/types";
+} from '@zilliz/milvus2-sdk-node/dist/milvus/types/Collection';
+import { DataType } from '@zilliz/milvus2-sdk-node/dist/milvus/types/Common';
+import { SearchParam } from '@zilliz/milvus2-sdk-node/dist/milvus/types';
 
 enum VectorTypes {
   Binary = DataType.BinaryVector,
@@ -36,7 +36,7 @@ export class CreateCollectionDto {
 
 export class ShowCollectionsDto {
   @IsOptional()
-  @IsEnum(ShowCollectionsType, { message: "Type allow all->0 inmemory->1" })
+  @IsEnum(ShowCollectionsType, { message: 'Type allow all->0 inmemory->1' })
   readonly type: ShowCollectionsType;
 }
 
@@ -48,6 +48,11 @@ export class InsertDataDto {
   readonly fields_data: any[];
 }
 
+export class ImportSampleDto {
+  readonly collection_name?: string;
+  readonly size: string;
+}
+
 export class VectorSearchDto {
   @IsOptional()
   partition_names?: string[];
@@ -67,7 +72,7 @@ export class VectorSearchDto {
   @IsOptional()
   output_fields?: string[];
 
-  @IsEnum(VectorTypes, { message: "Type allow all->0 inmemory->1" })
+  @IsEnum(VectorTypes, { message: 'Type allow all->0 inmemory->1' })
   vector_type: DataType.BinaryVector | DataType.FloatVector;
 }
 

server/src/utils/Helper.ts

@@ -1,4 +1,55 @@
 import { KeyValuePair } from '@zilliz/milvus2-sdk-node/dist/milvus/types/Common';
+import { FieldSchema } from '@zilliz/milvus2-sdk-node/dist/milvus/types/Response';
 
 export const findKeyValue = (obj: KeyValuePair[], key: string) =>
-  obj.find((v) => v.key === key)?.value;
+  obj.find(v => v.key === key)?.value;
+
+export const genDataByType = ({ data_type, type_params }: FieldSchema) => {
+  switch (data_type) {
+    case 'Bool':
+      return Math.random() > 0.5;
+    case 'Int8':
+      return Math.floor(Math.random() * 127);
+    case 'Int16':
+      return Math.floor(Math.random() * 32767);
+    case 'Int32':
+      return Math.floor(Math.random() * 214748364);
+    case 'Int64':
+      return Math.floor(Math.random() * 214748364);
+    case 'FloatVector':
+      return Array.from({ length: (type_params as any)[0].value }).map(() =>
+        Math.random()
+      );
+    case 'VarChar':
+      return makeRandomId((type_params as any)[0].value);
+  }
+};
+
+export const genRow = (fields: FieldSchema[]) => {
+  const result: any = {};
+  fields.forEach(field => {
+    if (!field.autoID) {
+      result[field.name] = genDataByType(field);
+    }
+  });
+  return result;
+};
+
+export const genRows = (fields: FieldSchema[], size: number) => {
+  const result = [];
+  for (let i = 0; i < size; i++) {
+    result[i] = genRow(fields);
+  }
+  return result;
+};
+
+export const makeRandomId = (length: number): string => {
+  var result = '';
+  var characters =
+    'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+  var charactersLength = characters.length;
+  for (var i = 0; i < length; i++) {
+    result += characters.charAt(Math.floor(Math.random() * charactersLength));
+  }
+  return result;
+};