Attu-384: refactor for performance (#425)

* cron job update part1

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* update task

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* upgrade task

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* part3

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* task part4

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* task part5

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* fix a bug

Signed-off-by: ruiyi.jiang <ruiyi.jiang@zilliz.com>

* task part7

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

---------

Signed-off-by: ryjiang <jiangruiyi@gmail.com>
Signed-off-by: ruiyi.jiang <ruiyi.jiang@zilliz.com>

client/src/App.tsx

@@ -13,19 +13,19 @@ import {
 function App() {
   return (
     <AuthProvider>
-      <RootProvider>
-        <SystemProvider>
-          <PrometheusProvider>
-            <DataProvider>
+      <DataProvider>
+        <RootProvider>
+          <SystemProvider>
+            <PrometheusProvider>
               <NavProvider>
                 <MuiPickersUtilsProvider utils={DayjsUtils}>
                   <Router></Router>
                 </MuiPickersUtilsProvider>
               </NavProvider>
-            </DataProvider>
-          </PrometheusProvider>
-        </SystemProvider>
-      </RootProvider>
+            </PrometheusProvider>
+          </SystemProvider>
+        </RootProvider>
+      </DataProvider>
     </AuthProvider>
   );
 }

client/src/components/grid/Table.tsx

@@ -1,4 +1,4 @@
-import { FC, forwardRef } from 'react';
+import { FC } from 'react';
 import { makeStyles } from '@material-ui/core/styles';
 import Table from '@material-ui/core/Table';
 import TableBody from '@material-ui/core/TableBody';

client/src/components/status/Types.ts

@@ -3,7 +3,7 @@ import { SchemaObject } from '@server/types';
 
 export type StatusType = {
   status: LOADING_STATE;
-  percentage?: string;
+  percentage?: number;
 };
 
 export type StatusActionType = {
@@ -12,7 +12,7 @@ export type StatusActionType = {
   action?: Function;
   field: SchemaObject;
   collectionName: string;
-  onIndexCreate: Function;
+  onIndexCreate?: Function;
 };
 
 // @todo need rename

client/src/context/Data.tsx

@@ -9,11 +9,13 @@ import {
 import { io, Socket } from 'socket.io-client';
 import { authContext } from '@/context';
 import { url, CollectionService, MilvusService, DatabaseService } from '@/http';
-import { checkIndexBuilding, checkLoading, getDbValueFromUrl } from '@/utils';
+import { IndexCreateParam, IndexManageParam } from '@/pages/schema/Types';
+import { getDbValueFromUrl } from '@/utils';
 import { DataContextType } from './Types';
-import { WS_EVENTS, WS_EVENTS_TYPE } from '@server/utils/Const';
 import { LAST_TIME_DATABASE } from '@/consts';
-import { CollectionObject } from '@server/types';
+import { CollectionObject, CollectionFullObject } from '@server/types';
+import { WS_EVENTS, WS_EVENTS_TYPE } from '@server/utils/Const';
+import { checkIndexing, checkLoading } from '@server/utils/Shared';
 
 export const dataContext = createContext<DataContextType>({
   loading: false,
@@ -25,6 +27,37 @@ export const dataContext = createContext<DataContextType>({
   setDatabaseList: () => {},
   fetchDatabases: async () => {},
   fetchCollections: async () => {},
+  fetchCollection: async () => {
+    return {} as CollectionFullObject;
+  },
+  createCollection: async () => {
+    return {} as CollectionFullObject;
+  },
+  loadCollection: async () => {
+    return {} as CollectionFullObject;
+  },
+  releaseCollection: async () => {
+    return {} as CollectionFullObject;
+  },
+  renameCollection: async () => {
+    return {} as CollectionFullObject;
+  },
+  duplicateCollection: async () => {
+    return {} as CollectionFullObject;
+  },
+  dropCollection: async () => {},
+  createIndex: async () => {
+    return {} as CollectionFullObject;
+  },
+  dropIndex: async () => {
+    return {} as CollectionFullObject;
+  },
+  createAlias: async () => {
+    return {} as CollectionFullObject;
+  },
+  dropAlias: async () => {
+    return {} as CollectionFullObject;
+  },
 });
 
 const { Provider } = dataContext;
@@ -48,44 +81,199 @@ export const DataProvider = (props: { children: React.ReactNode }) => {
   // socket ref
   const socket = useRef<Socket | null>(null);
 
-  // socket callback
-  const socketCallBack = useCallback(
+  // collection state test
+  const detectLoadingIndexing = useCallback(
     (collections: CollectionObject[]) => {
-      const hasLoadingOrBuildingCollection = collections.some(
-        v => checkLoading(v) || checkIndexBuilding(v)
-      );
-
-      setCollections(collections);
-      // If no collection is building index or loading collection
-      // stop server cron job
-      if (!hasLoadingOrBuildingCollection) {
+      const LoadingOrBuildingCollections = collections.filter(v => {
+        const isLoading = checkLoading(v);
+        const isBuildingIndex = checkIndexing(v);
+
+        return isLoading || isBuildingIndex;
+      });
+
+      // trigger cron if it has
+      if (LoadingOrBuildingCollections.length > 0) {
         MilvusService.triggerCron({
-          name: WS_EVENTS.COLLECTION,
-          type: WS_EVENTS_TYPE.STOP,
+          name: WS_EVENTS.COLLECTION_UPDATE,
+          type: WS_EVENTS_TYPE.START,
+          payload: {
+            database,
+            collections: LoadingOrBuildingCollections.map(
+              c => c.collection_name
+            ),
+          },
         });
       }
     },
     [database]
   );
 
-  // http fetch collection
+  // Websocket Callback: update single collection
+  const updateCollections = useCallback(
+    (updateCollections: CollectionFullObject[]) => {
+      // check state to see if it is loading or building index, if so, start server cron job
+      detectLoadingIndexing(updateCollections);
+      // update single collection
+      setCollections(prev => {
+        // update exsit collection
+        const newCollections = prev.map(v => {
+          const collectionToUpdate = updateCollections.find(c => c.id === v.id);
+
+          if (collectionToUpdate) {
+            return collectionToUpdate;
+          }
+
+          return v;
+        });
+
+        return newCollections;
+      });
+    },
+    [database]
+  );
+
+  // API: fetch databases
+  const fetchDatabases = async () => {
+    const res = await DatabaseService.getDatabases();
+
+    setDatabases(res.db_names);
+  };
+
+  // API:fetch collections
   const fetchCollections = async () => {
     try {
+      // set loading true
       setLoading(true);
+      // fetch collections
       const res = await CollectionService.getCollections();
+      // check state
+      detectLoadingIndexing(res);
+      // set collections
       setCollections(res);
+      // set loading false
       setLoading(false);
-    } catch (error) {
-      console.error(error);
     } finally {
       setLoading(false);
     }
   };
 
-  const fetchDatabases = async () => {
-    const res = await DatabaseService.getDatabases();
+  // API: fetch single collection
+  const fetchCollection = async (name: string) => {
+    // fetch collections
+    const res = await CollectionService.getCollection(name);
 
-    setDatabases(res.db_names);
+    // update collection
+    updateCollections([res]);
+
+    return res;
+  };
+
+  // API: create collection
+  const createCollection = async (data: any) => {
+    // create collection
+    const newCollection = await CollectionService.createCollection(data);
+    // inset collection to state
+    setCollections(prev => [...prev, newCollection]);
+
+    return newCollection;
+  };
+
+  // API: load collection
+  const loadCollection = async (name: string, param?: any) => {
+    // load collection
+    const newCollection = await CollectionService.loadCollection(name, param);
+    // update collection
+    updateCollections([newCollection]);
+
+    return newCollection;
+  };
+
+  // API: release collection
+  const releaseCollection = async (name: string) => {
+    // release collection
+    const newCollection = await CollectionService.releaseCollection(name);
+    // update collection
+    updateCollections([newCollection]);
+
+    return newCollection;
+  };
+
+  // API: rename collection
+  const renameCollection = async (name: string, newName: string) => {
+    // rename collection
+    const newCollection = await CollectionService.renameCollection(name, {
+      new_collection_name: newName,
+    });
+    updateCollections([newCollection]);
+
+    return newCollection;
+  };
+
+  // API: duplicate collection
+  const duplicateCollection = async (name: string, newName: string) => {
+    // duplicate collection
+    const newCollection = await CollectionService.duplicateCollection(name, {
+      new_collection_name: newName,
+    });
+    // inset collection to state
+    setCollections(prev => [...prev, newCollection]);
+
+    return newCollection;
+  };
+
+  // API: drop collection
+  const dropCollection = async (name: string) => {
+    // drop collection
+    const dropped = await CollectionService.dropCollection(name);
+    if (dropped.error_code === 'Success') {
+      // remove collection from state
+      setCollections(prev => prev.filter(v => v.collection_name !== name));
+    }
+  };
+
+  // API: create index
+  const createIndex = async (param: IndexCreateParam) => {
+    // create index
+    const newCollection = await CollectionService.createIndex(param);
+    // update collection
+    updateCollections([newCollection]);
+
+    return newCollection;
+  };
+
+  // API: drop index
+  const dropIndex = async (params: IndexManageParam) => {
+    // drop index
+    const { data } = await CollectionService.dropIndex(params);
+    // update collection
+    updateCollections([data]);
+
+    return data;
+  };
+
+  // API: create alias
+  const createAlias = async (collectionName: string, alias: string) => {
+    // create alias
+    const newCollection = await CollectionService.createAlias(collectionName, {
+      alias,
+    });
+    // update collection
+    updateCollections([newCollection]);
+
+    return newCollection;
+  };
+
+  // API: drop alias
+  const dropAlias = async (collectionName: string, alias: string) => {
+    // drop alias
+    const { data } = await CollectionService.dropAlias(collectionName, {
+      alias,
+    });
+
+    // update collection
+    updateCollections([data]);
+
+    return data;
   };
 
   useEffect(() => {
@@ -101,8 +289,6 @@ export const DataProvider = (props: { children: React.ReactNode }) => {
         console.log('--- ws connected ---', clientId);
         setConnected(true);
       });
-
-      socket.current?.on(WS_EVENTS.COLLECTION, socketCallBack);
     } else {
       socket.current?.disconnect();
       // clear collections
@@ -120,12 +306,18 @@ export const DataProvider = (props: { children: React.ReactNode }) => {
       setCollections([]);
       // remove all listeners
       socket.current?.offAny();
-      // listen to collection event
-      socket.current?.on(WS_EVENTS.COLLECTION, socketCallBack);
-      // get data
+      // listen to backend collection event
+      socket.current?.on(WS_EVENTS.COLLECTION_UPDATE, updateCollections);
+
+      // fetch db
       fetchCollections();
     }
-  }, [socketCallBack, connected]);
+
+    return () => {
+      // remove all listeners when component unmount
+      socket.current?.offAny();
+    };
+  }, [updateCollections, connected]);
 
   return (
     <Provider
@@ -139,6 +331,17 @@ export const DataProvider = (props: { children: React.ReactNode }) => {
         setDatabaseList: setDatabases,
         fetchDatabases,
         fetchCollections,
+        fetchCollection,
+        createCollection,
+        loadCollection,
+        releaseCollection,
+        renameCollection,
+        duplicateCollection,
+        dropCollection,
+        createIndex,
+        dropIndex,
+        createAlias,
+        dropAlias,
       }}
     >
       {props.children}

client/src/context/Root.tsx

@@ -116,7 +116,7 @@ export const RootProvider = (props: { children: React.ReactNode }) => {
     if (isAuth) {
       const fetchVersion = async () => {
         const res = await MilvusService.getVersion();
-        setVersionInfo(res);
+        setVersionInfo(res as any);
       };
       fetchVersion();
     }

client/src/context/Types.ts

@@ -1,6 +1,7 @@
 import { Dispatch, ReactElement, SetStateAction } from 'react';
-import { CollectionObject } from '@server/types';
+import { CollectionObject, CollectionFullObject } from '@server/types';
 import { NavInfo } from '@/router/Types';
+import { IndexCreateParam, IndexManageParam } from '@/pages/schema/Types';
 
 export type RootContextType = {
   openSnackBar: OpenSnackBarType;
@@ -98,4 +99,27 @@ export type DataContextType = {
   setDatabaseList: Dispatch<SetStateAction<string[]>>;
   fetchDatabases: () => Promise<void>;
   fetchCollections: () => Promise<void>;
+  fetchCollection: (name: string) => Promise<CollectionFullObject>;
+  createCollection: (data: any) => Promise<CollectionFullObject>;
+  loadCollection: (name: string, param?: any) => Promise<CollectionFullObject>;
+  releaseCollection: (name: string) => Promise<CollectionFullObject>;
+  renameCollection: (
+    name: string,
+    newName: string
+  ) => Promise<CollectionFullObject>;
+  duplicateCollection: (
+    name: string,
+    newName: string
+  ) => Promise<CollectionFullObject>;
+  dropCollection: (name: string) => Promise<void>;
+  createIndex: (param: IndexCreateParam) => Promise<CollectionFullObject>;
+  dropIndex: (params: IndexManageParam) => Promise<CollectionFullObject>;
+  createAlias: (
+    collectionName: string,
+    alias: string
+  ) => Promise<CollectionFullObject>;
+  dropAlias: (
+    collectionName: string,
+    alias: string
+  ) => Promise<CollectionFullObject>;
 };

client/src/hooks/Query.ts

@@ -115,7 +115,7 @@ export const useQuery = (params: {
 
   // get collection info
   const prepare = async (collectionName: string) => {
-    const collection = await CollectionService.getCollectionInfo(
+    const collection = await CollectionService.getCollection(
       collectionName
     );
     setFields([

client/src/http/BaseModel.ts

@@ -43,8 +43,6 @@ export default class BaseModel {
     } as any;
     if (timeout) httpConfig.timeout = timeout;
     const res = await http(httpConfig);
-    // conflict with collection view data structure, status is useless, so delete here.
-    delete res.data.data.status;
     return (res.data.data || {}) as T;
   }
 
@@ -69,13 +67,13 @@ export default class BaseModel {
 
     const res = await http.delete(path, { data: data });
 
-    return res.data;
+    return res.data as T;
   }
 
-  static async batchDelete(options: updateParamsType) {
+  static async batchDelete<T>(options: updateParamsType) {
     const { path, data } = options;
     const res = await http.post(path, data);
-    return res.data;
+    return res.data as T;
   }
 
   static async query(options: updateParamsType) {

client/src/http/Collection.service.ts

@@ -7,7 +7,12 @@ import {
   CollectionObject,
   CountObject,
   StatisticsObject,
+  ResStatus,
+  DescribeIndexRes,
+  IndexObject,
 } from '@server/types';
+import { ManageRequestMethods } from '../types/Common';
+import { IndexCreateParam, IndexManageParam } from '@/pages/schema/Types';
 
 export class CollectionService extends BaseModel {
   static getCollections(data?: {
@@ -16,7 +21,7 @@ export class CollectionService extends BaseModel {
     return super.findAll({ path: '/collections', params: data || {} });
   }
 
-  static getCollectionInfo(collectionName: string) {
+  static getCollection(collectionName: string) {
     return super.search<CollectionFullObject>({
       path: `/collections/${collectionName}`,
       params: {},
@@ -24,22 +29,22 @@ export class CollectionService extends BaseModel {
   }
 
   static createCollection(data: any) {
-    return super.create({ path: `collections`, data });
+    return super.create<CollectionFullObject>({ path: `collections`, data });
   }
 
-  static deleteCollection(collectionName: string) {
-    return super.delete({ path: `/collections/${collectionName}` });
+  static dropCollection(collectionName: string) {
+    return super.delete<ResStatus>({ path: `/collections/${collectionName}` });
   }
 
   static loadCollection(collectionName: string, param?: LoadReplicaReq) {
-    return super.update({
+    return super.update<CollectionFullObject>({
       path: `/collections/${collectionName}/load`,
       data: param,
     });
   }
 
   static releaseCollection(collectionName: string) {
-    return super.update({
+    return super.update<CollectionFullObject>({
       path: `/collections/${collectionName}/release`,
     });
   }
@@ -48,17 +53,17 @@ export class CollectionService extends BaseModel {
     collectionName: string,
     params: { new_collection_name: string }
   ) {
-    return super.create({
+    return super.create<CollectionFullObject>({
       path: `/collections/${collectionName}`,
       data: params,
     });
   }
 
-  static duplicate(
+  static duplicateCollection(
     collectionName: string,
     params: { new_collection_name: string }
   ) {
-    return super.create({
+    return super.create<CollectionFullObject>({
       path: `/collections/${collectionName}/duplicate`,
       data: params,
     });
@@ -79,18 +84,52 @@ export class CollectionService extends BaseModel {
   }
 
   static createAlias(collectionName: string, params: { alias: string }) {
-    return super.create({
+    return super.create<CollectionFullObject>({
       path: `/collections/${collectionName}/alias`,
       data: params,
     });
   }
 
   static dropAlias(collectionName: string, params: { alias: string }) {
-    return super.delete({
+    return super.delete<{ data: CollectionFullObject }>({
       path: `/collections/${collectionName}/alias/${params.alias}`,
     });
   }
 
+  static async describeIndex(collectionName: string): Promise<IndexObject[]> {
+    const res = await super.findAll<DescribeIndexRes>({
+      path: `/collections/index`,
+      params: { collection_name: collectionName },
+    });
+    return res.index_descriptions;
+  }
+
+  static async createIndex(param: IndexCreateParam) {
+    const path = `/collections/index`;
+    const type: ManageRequestMethods = ManageRequestMethods.CREATE;
+
+    return super.create<CollectionFullObject>({
+      path,
+      data: { ...param, type },
+    });
+  }
+
+  static async dropIndex(param: IndexManageParam) {
+    const path = `/collections/index`;
+    const type: ManageRequestMethods = ManageRequestMethods.DELETE;
+
+    return super.batchDelete<{ data: CollectionFullObject }>({
+      path,
+      data: { ...param, type },
+    });
+  }
+
+  static async flush() {
+    const path = `/collections/index/flush`;
+
+    return super.query({ path, data: {} });
+  }
+
   static queryData(collectionName: string, params: QueryParam) {
     return super.query({
       path: `/collections/${collectionName}/query`,

client/src/http/Index.service.ts

@@ -1,37 +0,0 @@
-import { IndexCreateParam, IndexManageParam } from '../pages/schema/Types';
-import { ManageRequestMethods } from '../types/Common';
-import BaseModel from './BaseModel';
-import { DescribeIndexRes, IndexObject } from '@server/types';
-
-export class IndexService extends BaseModel {
-  static async describeIndex(collectionName: string): Promise<IndexObject[]> {
-    const res = await super.findAll<DescribeIndexRes>({
-      path: `/schema/index`,
-      params: { collection_name: collectionName },
-    });
-    return res.index_descriptions;
-  }
-
-  static async createIndex(param: IndexCreateParam) {
-    const path = `/schema/index`;
-    const type: ManageRequestMethods = ManageRequestMethods.CREATE;
-
-    return super.create({
-      path,
-      data: { ...param, type },
-    });
-  }
-
-  static async deleteIndex(param: IndexManageParam) {
-    const path = `/schema/index`;
-    const type: ManageRequestMethods = ManageRequestMethods.DELETE;
-
-    return super.batchDelete({ path, data: { ...param, type } });
-  }
-
-  static async flush() {
-    const path = `/schema/index/flush`;
-
-    return super.query({ path, data: {} });
-  }
-}

client/src/http/Milvus.service.ts

@@ -1,5 +1,5 @@
-import { WS_EVENTS, WS_EVENTS_TYPE } from '@server/utils/Const';
 import BaseModel from './BaseModel';
+import { CronJobObject } from '@server/types';
 
 export class MilvusService extends BaseModel {
   static connect(data: {
@@ -37,7 +37,7 @@ export class MilvusService extends BaseModel {
     });
   }
 
-  static triggerCron(data: { name: WS_EVENTS; type: WS_EVENTS_TYPE }) {
+  static triggerCron(data: CronJobObject) {
     return super.update({
       path: '/crons',
       data,

client/src/http/index.ts

@@ -4,7 +4,6 @@ export * from './BaseModel';
 
 // service
 export * from './Collection.service';
-export * from './Index.service';
 export * from './Partition.service';
 export * from './Data.service';
 export * from './Milvus.service';

client/src/pages/collections/Aliases.tsx

@@ -1,13 +1,12 @@
 import { useContext } from 'react';
 import { Chip, IconButton, makeStyles, Theme } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
-import { rootContext } from '@/context';
+import { rootContext, dataContext } from '@/context';
 import { AliasesProps } from './Types';
 import icons from '@/components/icons/Icons';
 import DeleteIcon from '@material-ui/icons/Delete';
 import CreateAliasDialog from '../dialogs/CreateAliasDialog';
 import DeleteTemplate from '@/components/customDialog/DeleteDialogTemplate';
-import { CollectionService } from '@/http';
 
 const useStyles = makeStyles((theme: Theme) => ({
   wrapper: {
@@ -23,6 +22,8 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 export default function Aliases(props: AliasesProps) {
+  const { dropAlias } = useContext(dataContext);
+
   const {
     aliases,
     collectionName,
@@ -48,8 +49,8 @@ export default function Aliases(props: AliasesProps) {
         component: (
           <CreateAliasDialog
             collectionName={collectionName}
-            cb={() => {
-              onCreate();
+            cb={async collectionName => {
+              await onCreate(collectionName);
             }}
           />
         ),
@@ -77,12 +78,10 @@ export default function Aliases(props: AliasesProps) {
     collection: string;
     alias: string;
   }) => {
-    await CollectionService.dropAlias(params.collection, {
-      alias: params.alias,
-    });
+    await dropAlias(params.collection, params.alias);
     openSnackBar(successTrans('delete', { name: collectionTrans('alias') }));
     handleCloseDialog();
-    onDelete();
+    await onDelete(collectionName);
   };
 
   const _onDelete = (alias: { collection: string; alias: string }) => {

client/src/pages/collections/Collections.tsx

@@ -4,7 +4,7 @@ import { makeStyles, Theme, Chip, Tooltip } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
 import Highlighter from 'react-highlight-words';
 import { rootContext, authContext, dataContext } from '@/context';
-import { IndexService } from '@/http';
+import { CollectionService } from '@/http';
 import { useNavigationHook, usePaginationHook } from '@/hooks';
 import { ALL_ROUTER_TYPES } from '@/router/Types';
 import AttuGrid from '@/components/grid/Grid';
@@ -71,7 +71,7 @@ const useStyles = makeStyles((theme: Theme) => ({
 const Collections = () => {
   useNavigationHook(ALL_ROUTER_TYPES.COLLECTIONS);
   const { isManaged } = useContext(authContext);
-  const { collections, database, loading, fetchCollections } =
+  const { collections, database, loading, fetchCollections, fetchCollection } =
     useContext(dataContext);
 
   const [searchParams] = useSearchParams();
@@ -102,7 +102,7 @@ const Collections = () => {
   };
 
   const clearIndexCache = useCallback(async () => {
-    await IndexService.flush();
+    await CollectionService.flush();
   }, []);
 
   const formatCollections = useMemo(() => {
@@ -143,7 +143,6 @@ const Collections = () => {
                       name: collectionTrans('collection'),
                     })
                   );
-                  await fetchCollections();
                 }}
               />
             ),
@@ -208,7 +207,6 @@ const Collections = () => {
                     })
                   );
                   setSelectedCollections([]);
-                  await fetchCollections();
                 }}
               />
             ),
@@ -243,8 +241,10 @@ const Collections = () => {
                 }
                 // user can't select partition on collection page, so default value is ''
                 defaultSelectedPartition={''}
-                onInsert={async () => {
-                  await fetchCollections();
+                onInsert={async (collectionName: string) => {
+                  setTimeout(async () => {
+                    await fetchCollection(collectionName);
+                  });
                   setSelectedCollections([]);
                 }}
               />
@@ -272,13 +272,12 @@ const Collections = () => {
           params: {
             component: (
               <RenameCollectionDialog
-                cb={async () => {
+                cb={async (collectionName: string) => {
                   openSnackBar(
                     successTrans('rename', {
                       name: collectionTrans('collection'),
                     })
                   );
-                  await fetchCollections();
                   setSelectedCollections([]);
                 }}
                 collectionName={selectedCollections[0].collection_name}
@@ -309,7 +308,6 @@ const Collections = () => {
                     })
                   );
                   setSelectedCollections([]);
-                  await fetchCollections();
                 }}
                 collectionName={selectedCollections[0].collection_name}
                 collections={collections}
@@ -339,7 +337,6 @@ const Collections = () => {
                       name: collectionTrans('collection'),
                     })
                   );
-                  await fetchCollections();
                   setSelectedCollections([]);
                 }}
                 collections={selectedCollections}
@@ -362,6 +359,9 @@ const Collections = () => {
         clearIndexCache();
         fetchCollections();
       },
+      disabled: () => {
+        return loading;
+      },
       label: btnTrans('refresh'),
     },
 
@@ -402,13 +402,12 @@ const Collections = () => {
       id: 'status',
       align: 'left',
       disablePadding: false,
-      sortBy: 'status',
+      sortBy: 'loadedPercentage',
       label: collectionTrans('status'),
       formatter(v) {
         return (
           <StatusAction
             status={v.status}
-            onIndexCreate={fetchCollections}
             percentage={v.loadedPercentage}
             field={v.schema}
             collectionName={v.collection_name}
@@ -420,26 +419,24 @@ const Collections = () => {
                   component:
                     v.status === LOADING_STATE.UNLOADED ? (
                       <LoadCollectionDialog
-                        collection={v.collection_name}
-                        onLoad={async () => {
+                        collectionName={v.collection_name}
+                        onLoad={async (collectionName: string) => {
                           openSnackBar(
                             successTrans('load', {
                               name: collectionTrans('collection'),
                             })
                           );
-                          await fetchCollections();
                         }}
                       />
                     ) : (
                       <ReleaseCollectionDialog
-                        collection={v.collection_name}
-                        onRelease={async () => {
+                        collectionName={v.collection_name}
+                        onRelease={async (collectionName: string) => {
                           openSnackBar(
                             successTrans('release', {
                               name: collectionTrans('collection'),
                             })
                           );
-                          await fetchCollections();
                         }}
                       />
                     ),
@@ -472,7 +469,7 @@ const Collections = () => {
                 />
               </Tooltip>
             ) : null}
-            {v.schema.enable_dynamic_field ? (
+            {v.schema && v.schema.enable_dynamic_field ? (
               <Tooltip
                 title={collectionTrans('dynamicSchemaTooltip')}
                 placement="top"
@@ -550,7 +547,14 @@ const Collections = () => {
               type: 'custom',
               params: {
                 component: (
-                  <ImportSampleDialog collection={row.collection_name} />
+                  <ImportSampleDialog
+                    collection={row.collection_name}
+                    cb={async (collectionName: string) => {
+                      setTimeout(async () => {
+                        await fetchCollection(collectionName);
+                      });
+                    }}
+                  />
                 ),
               },
             });
@@ -580,12 +584,7 @@ const Collections = () => {
       ),
       formatter(v) {
         return (
-          <Aliases
-            aliases={v.aliases}
-            collectionName={v.collection_name}
-            onCreate={fetchCollections}
-            onDelete={fetchCollections}
-          />
+          <Aliases aliases={v.aliases} collectionName={v.collection_name} />
         );
       },
     });
@@ -610,7 +609,7 @@ const Collections = () => {
           colDefinitions={colDefinitions}
           rows={collectionList}
           rowCount={total}
-          primaryKey="collection_name"
+          primaryKey="id"
           selected={selectedCollections}
           setSelected={handleSelectChange}
           page={currentPage}

client/src/pages/collections/StatusAction.tsx

@@ -132,7 +132,6 @@ const StatusAction: FC<StatusActionType> = props => {
 
   // UI state
   const collectionLoaded = status === LOADING_STATE.LOADED;
-
   return (
     <div className={classes.root}>
       {field.hasVectorIndex && (
@@ -153,9 +152,8 @@ const StatusAction: FC<StatusActionType> = props => {
         </Tooltip>
       )}
       <IndexTypeElement
-        data={field.vectorFields[0]!}
+        field={field.vectorFields[0]!}
         collectionName={collectionName}
-        cb={() => onIndexCreate()}
         disabled={collectionLoaded}
         disabledTooltip={collectionTrans('releaseCollectionFirst')}
       />

client/src/pages/databases/Databases.tsx

@@ -21,13 +21,15 @@ const useStyles = makeStyles((theme: Theme) => ({
   wrapper: {
     flexDirection: 'row',
     gap: theme.spacing(2),
+    maxHeight: 'calc(100% - 105px)', // header + margin
   },
-  card: {
+  tree: {
     boxShadow: 'none',
     flexBasis: theme.spacing(28),
     width: theme.spacing(28),
     flexGrow: 0,
     flexShrink: 0,
+    overflow: 'auto',
   },
   tab: {
     flexGrow: 1,
@@ -169,7 +171,7 @@ const Databases = () => {
   const uiLoading = localLoading || loading;
   return (
     <section className={`page-wrapper ${classes.wrapper}`}>
-      <section className={classes.card}>
+      <section className={classes.tree}>
         {uiLoading ? (
           `loading`
         ) : (

client/src/pages/dialogs/CreateAliasDialog.tsx

@@ -1,13 +1,12 @@
 import { FC, useContext, useMemo, useState } from 'react';
 import { Typography, makeStyles, Theme } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
-import { rootContext } from '@/context';
+import { rootContext, dataContext } from '@/context';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
 import CustomInput from '@/components/customInput/CustomInput';
 import { formatForm } from '@/utils';
 import { useFormValidation } from '@/hooks';
 import { ITextfieldConfig } from '@/components/customInput/Types';
-import { CollectionService } from '@/http';
 import { CreateAliasProps } from './Types';
 
 const useStyles = makeStyles((theme: Theme) => ({
@@ -17,6 +16,9 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 const CreateAliasDialog: FC<CreateAliasProps> = props => {
+  const { createAlias } = useContext(dataContext);
+  const { handleCloseDialog } = useContext(rootContext);
+
   const { cb, collectionName } = props;
   const [form, setForm] = useState({
     alias: '',
@@ -31,7 +33,6 @@ const CreateAliasDialog: FC<CreateAliasProps> = props => {
 
   const { validation, checkIsValid, disabled } = useFormValidation(checkedForm);
 
-  const { handleCloseDialog } = useContext(rootContext);
   const { t: dialogTrans } = useTranslation('dialog');
   const { t: warningTrans } = useTranslation('warning');
   const { t: collectionTrans } = useTranslation('collection');
@@ -42,9 +43,9 @@ const CreateAliasDialog: FC<CreateAliasProps> = props => {
   };
 
   const handleConfirm = async () => {
-    await CollectionService.createAlias(collectionName, form);
+    await createAlias(collectionName, form.alias);
     handleCloseDialog();
-    cb && cb();
+    cb && (await cb(collectionName));
   };
 
   const aliasInputConfig: ITextfieldConfig = {

client/src/pages/dialogs/CreateCollectionDialog.tsx

@@ -10,12 +10,11 @@ import DialogTemplate from '@/components/customDialog/DialogTemplate';
 import CustomInput from '@/components/customInput/CustomInput';
 import CustomSelector from '@/components/customSelector/CustomSelector';
 import { ITextfieldConfig } from '@/components/customInput/Types';
-import { rootContext } from '@/context';
+import { rootContext, dataContext } from '@/context';
 import { useFormValidation } from '@/hooks';
 import { formatForm, TypeEnum } from '@/utils';
 import { DataTypeEnum, ConsistencyLevelEnum, DEFAULT_ATTU_DIM } from '@/consts';
 import CreateFields from '../collections/CreateFields';
-import { CollectionService } from '@/http';
 import {
   CollectionCreateParam,
   CollectionCreateProps,
@@ -58,6 +57,7 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
+  const { createCollection } = useContext(dataContext);
   const classes = useStyles();
   const { handleCloseDialog } = useContext(rootContext);
   const { t: collectionTrans } = useTranslation('collection');
@@ -242,7 +242,7 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
       consistency_level: consistencyLevel,
     };
 
-    await CollectionService.createCollection({
+    await createCollection({
       ...param,
     });
 

client/src/pages/dialogs/DropCollectionDialog.tsx

@@ -1,24 +1,24 @@
 import { FC, useContext } from 'react';
 import { useTranslation } from 'react-i18next';
-import { rootContext } from '@/context';
+import { rootContext, dataContext } from '@/context';
 import DeleteTemplate from '@/components/customDialog/DeleteDialogTemplate';
-import { CollectionService } from '@/http';
 import { DropCollectionProps } from './Types';
 
 const DropCollectionDialog: FC<DropCollectionProps> = props => {
   const { collections, onDelete } = props;
   const { handleCloseDialog } = useContext(rootContext);
+  const { dropCollection } = useContext(dataContext);
   const { t: collectionTrans } = useTranslation('collection');
   const { t: btnTrans } = useTranslation('btn');
   const { t: dialogTrans } = useTranslation('dialog');
 
   const handleDelete = async () => {
     for (const item of collections) {
-      await CollectionService.deleteCollection(item.collection_name);
+      await dropCollection(item.collection_name);
     }
 
     handleCloseDialog();
-    onDelete && onDelete();
+    onDelete && (await onDelete());
   };
 
   return (

client/src/pages/dialogs/DuplicateCollectionDailog.tsx

@@ -1,13 +1,12 @@
 import { FC, useContext, useMemo, useState } from 'react';
 import { Typography, makeStyles, Theme } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
-import { rootContext } from '@/context';
+import { rootContext, dataContext } from '@/context';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
 import CustomInput from '@/components/customInput/CustomInput';
 import { formatForm } from '@/utils';
 import { useFormValidation } from '@/hooks';
 import { ITextfieldConfig } from '@/components/customInput/Types';
-import { CollectionService } from '@/http';
 import { DuplicateCollectionDialogProps } from './Types';
 
 const useStyles = makeStyles((theme: Theme) => ({
@@ -20,6 +19,8 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 const DuplicateCollectionDialog: FC<DuplicateCollectionDialogProps> = props => {
+  const { duplicateCollection } = useContext(dataContext);
+
   const { cb, collectionName, collections } = props;
   const [form, setForm] = useState({
     duplicate: `${collectionName}_duplicate`,
@@ -46,12 +47,10 @@ const DuplicateCollectionDialog: FC<DuplicateCollectionDialogProps> = props => {
 
   const handleConfirm = async () => {
     // duplicate
-    await CollectionService.duplicate(collectionName, {
-      new_collection_name: form.duplicate,
-    });
+    await duplicateCollection(collectionName, form.duplicate);
     // close dialog
     handleCloseDialog();
-    cb && cb();
+    cb && (await cb(form.duplicate));
   };
 
   const duplicateInputConfig: ITextfieldConfig = {
@@ -75,8 +74,10 @@ const DuplicateCollectionDialog: FC<DuplicateCollectionDialogProps> = props => {
       {
         rule: 'custom',
         extraParam: {
-          compare: (value) => {
-            return !collections.some(collection => collection.collection_name === value);
+          compare: value => {
+            return !collections.some(
+              collection => collection.collection_name === value
+            );
           },
         },
         errorText: collectionTrans('duplicateNameExist'),

client/src/pages/dialogs/ImportSampleDialog.tsx

@@ -119,7 +119,7 @@ const ImportSampleDialog: FC<{ collection: string; cb?: Function }> = props => {
       }
       await DataService.flush(collectionName);
       if (props.cb) {
-        props.cb();
+        await props.cb(collectionName);
       }
       return { result: true, msg: '' };
     } catch (err: any) {

client/src/pages/dialogs/LoadCollectionDialog.tsx

@@ -7,8 +7,8 @@ import {
   FormControlLabel,
 } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
-import { authContext, rootContext } from '@/context';
-import { CollectionService, MilvusService } from '@/http';
+import { authContext, rootContext, dataContext } from '@/context';
+import { MilvusService } from '@/http';
 import { useFormValidation } from '@/hooks';
 import { formatForm, parseJson, getNode } from '@/utils';
 import { MILVUS_NODE_TYPE, MILVUS_DEPLOY_MODE } from '@/consts';
@@ -17,7 +17,6 @@ import { ITextfieldConfig } from '@/components/customInput/Types';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
 import CustomToolTip from '@/components/customToolTip/CustomToolTip';
 import icons from '@/components/icons/Icons';
-import { WS_EVENTS, WS_EVENTS_TYPE } from '@server/utils/Const';
 
 const useStyles = makeStyles((theme: Theme) => ({
   desc: {
@@ -38,8 +37,9 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 const LoadCollectionDialog = (props: any) => {
+  const { loadCollection } = useContext(dataContext);
   const classes = useStyles();
-  const { collection, onLoad } = props;
+  const { collectionName, onLoad } = props;
   const { t: dialogTrans } = useTranslation('dialog');
   const { t: collectionTrans } = useTranslation('collection');
   const { t: btnTrans } = useTranslation('btn');
@@ -99,16 +99,11 @@ const LoadCollectionDialog = (props: any) => {
     }
 
     // load collection request
-    await CollectionService.loadCollection(collection, params);
-
-    MilvusService.triggerCron({
-      name: WS_EVENTS.COLLECTION,
-      type: WS_EVENTS_TYPE.START,
-    });
+    await loadCollection(collectionName, params);
 
     // callback
     if (onLoad) {
-      await onLoad();
+      await onLoad(collectionName);
     }
     // close dialog
     handleCloseDialog();
@@ -170,7 +165,7 @@ const LoadCollectionDialog = (props: any) => {
   return (
     <DialogTemplate
       title={dialogTrans('loadTitle', {
-        type: collection,
+        type: collectionName,
       })}
       handleClose={handleCloseDialog}
       children={

client/src/pages/dialogs/ReleaseCollectionDialog.tsx

@@ -2,8 +2,7 @@ import { useContext, useState } from 'react';
 import { Typography, makeStyles, Theme } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
-import { CollectionService } from '@/http';
-import { rootContext } from '@/context';
+import { rootContext, dataContext } from '@/context';
 
 const useStyles = makeStyles((theme: Theme) => ({
   desc: {
@@ -13,9 +12,11 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 const ReleaseCollectionDialog = (props: any) => {
+  const { releaseCollection } = useContext(dataContext);
+
   const classes = useStyles();
 
-  const { collection, onRelease } = props;
+  const { collectionName, onRelease } = props;
   const { t: dialogTrans } = useTranslation('dialog');
   const { t: btnTrans } = useTranslation('btn');
   const { handleCloseDialog } = useContext(rootContext);
@@ -27,9 +28,10 @@ const ReleaseCollectionDialog = (props: any) => {
     setDisabled(true);
     try {
       // release collection
-      await CollectionService.releaseCollection(collection);
+      await releaseCollection(collectionName);
+
       // execute callback
-      onRelease && onRelease();
+      onRelease && (await onRelease(collectionName));
       // enable confirm button
       setDisabled(false);
       // close dialog
@@ -43,13 +45,13 @@ const ReleaseCollectionDialog = (props: any) => {
   return (
     <DialogTemplate
       title={dialogTrans('releaseTitle', {
-        type: collection,
+        type: collectionName,
       })}
       handleClose={handleCloseDialog}
       children={
         <>
           <Typography variant="body1" component="p" className={classes.desc}>
-            {dialogTrans('releaseContent', { type: collection })}
+            {dialogTrans('releaseContent', { type: collectionName })}
           </Typography>
         </>
       }

client/src/pages/dialogs/RenameCollectionDialog.tsx

@@ -1,10 +1,9 @@
 import { FC, useContext, useMemo, useState } from 'react';
 import { Typography, makeStyles, Theme } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
-import { rootContext } from '@/context';
+import { rootContext, dataContext } from '@/context';
 import { formatForm } from '@/utils';
 import { useFormValidation } from '@/hooks';
-import { CollectionService } from '@/http';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
 import CustomInput from '@/components/customInput/CustomInput';
 import { ITextfieldConfig } from '@/components/customInput/Types';
@@ -17,6 +16,8 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 const RenameCollectionDialog: FC<RenameCollectionProps> = props => {
+  const { renameCollection } = useContext(dataContext);
+
   const { collectionName, cb } = props;
   const [form, setForm] = useState({
     new_collection_name: '',
@@ -42,9 +43,9 @@ const RenameCollectionDialog: FC<RenameCollectionProps> = props => {
   };
 
   const handleConfirm = async () => {
-    await CollectionService.renameCollection(collectionName, form);
+    await renameCollection(collectionName, form.new_collection_name);
     handleCloseDialog();
-    cb && cb();
+    cb && (await cb(form.new_collection_name));
   };
 
   const renameInputCfg: ITextfieldConfig = {

client/src/pages/dialogs/Types.ts

@@ -12,7 +12,7 @@ export interface DropPartitionProps {
 }
 
 export interface PartitionCreateProps {
-  onCreate: () => void;
+  onCreate: (collectionName: string) => void;
   collectionName: string;
 }
 
@@ -26,7 +26,7 @@ export interface FlushDialogProps extends CollectionDialogBaseProps {}
 
 export interface CreateAliasProps {
   collectionName: string;
-  cb?: () => void;
+  cb?: (collectionName: string) => void;
 }
 
 export interface EmptyDataProps {
@@ -40,7 +40,7 @@ export interface DuplicateCollectionDialogProps extends CreateAliasProps {
 
 export interface RenameCollectionProps {
   collectionName: string;
-  cb?: () => void;
+  cb?: (collectionName: string) => void;
 }
 export interface LoadSampleParam {
   collection_name: string;

client/src/pages/dialogs/insert/Dialog.tsx

@@ -341,7 +341,7 @@ const InsertContainer: FC<InsertContentProps> = ({
       await DataService.insertData(collectionValue, param);
       await DataService.flush(collectionValue);
       // update collections
-      onInsert();
+      await onInsert(collectionValue);
       setInsertStatus(InsertStatusEnum.success);
     } catch (err: any) {
       const {

client/src/pages/overview/Overview.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useContext, useEffect, useMemo, useState } from 'react';
+import { useContext, useMemo } from 'react';
 import {
   makeStyles,
   Theme,
@@ -15,12 +15,10 @@ import EmptyCard from '@/components/cards/EmptyCard';
 import icons from '@/components/icons/Icons';
 import { LOADING_STATE, MILVUS_DEPLOY_MODE } from '@/consts';
 import { useNavigationHook } from '@/hooks';
-import { CollectionService } from '@/http';
 import { ALL_ROUTER_TYPES } from '@/router/Types';
 import { formatNumber } from '@/utils';
 import CollectionCard from './collectionCard/CollectionCard';
 import StatisticsCard from './statisticsCard/StatisticsCard';
-import { StatisticsObject } from '@server/types';
 
 const useStyles = makeStyles((theme: Theme) => ({
   overviewContainer: {
@@ -115,34 +113,16 @@ const Overview = () => {
   const { t: overviewTrans } = useTranslation('overview');
   const { t: collectionTrans } = useTranslation('collection');
   const { t: successTrans } = useTranslation('success');
-  const [statistics, setStatistics] = useState<StatisticsObject>({
-    collectionCount: 0,
-    totalData: 0,
-  });
-  const [loadingLocal, setLoadingLocal] = useState(false);
   const { openSnackBar } = useContext(rootContext);
 
-  const fetchData = useCallback(async () => {
-    if (loading) return;
-    setLoadingLocal(true);
-    const res = await CollectionService.getStatistics();
-    setStatistics(res);
-    setLoadingLocal(false);
-  }, [database, collections]);
-
-  useEffect(() => {
-    fetchData();
-  }, [fetchData]);
-
   const loadCollections = collections.filter(
-    c => c.status !== LOADING_STATE.UNLOADED
+    c => c.status !== LOADING_STATE.UNLOADED && typeof c.status !== 'undefined'
   );
 
   const onRelease = () => {
     openSnackBar(
       successTrans('release', { name: collectionTrans('collection') })
     );
-    fetchData();
   };
 
   const statisticsData = useMemo(() => {
@@ -155,19 +135,24 @@ const Overview = () => {
         },
         {
           label: overviewTrans('all'),
-          value: formatNumber(statistics.collectionCount),
+          value: collections.length,
           valueColor: theme.palette.primary.main,
         },
         {
           label: overviewTrans('data'),
           value: overviewTrans('rows', {
-            number: formatNumber(statistics.totalData),
+            number: formatNumber(
+              collections.reduce(
+                (acc, cur) => acc + Number(cur.rowCount || 0),
+                0
+              )
+            ),
           }) as string,
           valueColor: theme.palette.primary.dark,
         },
       ],
     };
-  }, [overviewTrans, statistics, loadCollections]);
+  }, [overviewTrans, loadCollections]);
 
   const CollectionIcon = icons.navCollection;
 
@@ -204,8 +189,6 @@ const Overview = () => {
     return `${duration.toFixed(2)} ${unit}`;
   }, [data.rootCoord]);
 
-  const _loading = loadingLocal || loading;
-
   return (
     <section className={`page-wrapper  ${classes.overviewContainer}`}>
       <section className={classes.dbWrapper}>
@@ -223,20 +206,18 @@ const Overview = () => {
             {loadCollections.map(collection => (
               <CollectionCard
                 key={collection.id}
-                data={collection}
+                collection={collection}
                 onRelease={onRelease}
               />
             ))}
           </div>
         ) : (
           <EmptyCard
-            loading={_loading}
+            loading={loading}
             wrapperClass={classes.emptyCard}
-            icon={!_loading ? <CollectionIcon /> : undefined}
+            icon={!loading ? <CollectionIcon /> : undefined}
             text={
-              _loading
-                ? overviewTrans('loading')
-                : collectionTrans('noLoadData')
+              loading ? overviewTrans('loading') : collectionTrans('noLoadData')
             }
           />
         )}

client/src/pages/overview/collectionCard/CollectionCard.tsx

@@ -83,7 +83,7 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 const CollectionCard: FC<CollectionCardProps> = ({
-  data,
+  collection,
   onRelease,
   wrapperClass = '',
 }) => {
@@ -93,7 +93,12 @@ const CollectionCard: FC<CollectionCardProps> = ({
   const classes = useStyles();
   const { setDialog } = useContext(rootContext);
 
-  const { collection_name, status: status, loadedPercentage, replicas } = data;
+  const {
+    collection_name,
+    status: status,
+    loadedPercentage,
+    replicas,
+  } = collection;
   const navigate = useNavigate();
   // icons
   const RightArrowIcon = icons.rightArrow;
@@ -125,35 +130,27 @@ const CollectionCard: FC<CollectionCardProps> = ({
     });
   };
 
-  useEffect(() => {
-    const fetchData = async () => {
-      try {
-        setLoading(true);
-        if (status === LOADING_STATE.LOADED) {
-          const data = await CollectionService.count(collection_name);
-          setCount(data.rowCount);
-        }
-      } catch (e) {
-      } finally {
-        setLoading(false);
-      }
-    };
-
-    let exiting = false;
+  const fetchData = async () => {
+    try {
+      setLoading(true);
+      const data = await CollectionService.count(collection_name);
+      setCount(data.rowCount);
+    } catch (e) {
+    } finally {
+      setLoading(false);
+    }
+  };
 
-    if (!exiting) {
+  useEffect(() => {
+    if (status === LOADING_STATE.LOADED) {
       fetchData();
     }
-
-    return () => {
-      exiting = true;
-    };
-  }, [status, database]);
+  }, [status]);
 
   return (
     <Card
       className={`card-wrapper ${classes.wrapper} ${wrapperClass} ${
-        data.status === LOADING_STATE.LOADING && classes.loading
+        collection.status === LOADING_STATE.LOADING && classes.loading
       }`}
     >
       <CardContent>

client/src/pages/overview/collectionCard/Types.ts

@@ -1,7 +1,7 @@
 import { CollectionObject } from '@server/types';
 
 export interface CollectionCardProps {
-  data: CollectionObject;
+  collection: CollectionObject;
   onRelease: () => void;
   wrapperClass?: string;
 }

client/src/pages/partitions/Partitions.tsx

@@ -69,7 +69,7 @@ const Partitions = () => {
   };
 
   const fetchCollectionDetail = async (name: string) => {
-    const res = await CollectionService.getCollectionInfo(name);
+    const res = await CollectionService.getCollection(name);
     return res;
   };
 

client/src/pages/schema/IndexTypeElement.tsx

@@ -1,22 +1,13 @@
-import {
-  FC,
-  useContext,
-  useEffect,
-  useMemo,
-  useState,
-  MouseEvent,
-} from 'react';
+import { FC, useContext, MouseEvent } from 'react';
 import { useTranslation } from 'react-i18next';
 import Chip from '@material-ui/core/Chip';
 import { makeStyles, Theme, Tooltip } from '@material-ui/core';
 import { IndexCreateParam, IndexExtraParam, IndexManageParam } from './Types';
-import { IndexService } from '@/http';
-import { rootContext } from '@/context';
+import { rootContext, dataContext } from '@/context';
 import icons from '@/components/icons/Icons';
 import DeleteTemplate from '@/components/customDialog/DeleteDialogTemplate';
 import StatusIcon from '@/components/status/StatusIcon';
 import { ChildrenStatusType } from '@/components/status/Types';
-import { sleep } from '@/utils';
 import { IndexState } from '@/types/Milvus';
 import { NONE_INDEXABLE_DATA_TYPES, DataTypeStringEnum } from '@/consts';
 import CreateIndex from './Create';
@@ -67,15 +58,16 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 const IndexTypeElement: FC<{
-  data: FieldObject;
+  field: FieldObject;
   collectionName: string;
   disabled?: boolean;
   disabledTooltip?: string;
-  cb: (collectionName: string) => void;
-}> = ({ data, collectionName, cb, disabled, disabledTooltip }) => {
+  cb?: (collectionName: string) => void;
+}> = ({ field, collectionName, cb, disabled, disabledTooltip }) => {
+  const { createIndex, dropIndex } = useContext(dataContext);
+
   const classes = useStyles();
   // set empty string as default status
-  const [status, setStatus] = useState<string>(IndexState.Default);
   const { t: indexTrans } = useTranslation('index');
   const { t: btnTrans } = useTranslation('btn');
   const { t: dialogTrans } = useTranslation('dialog');
@@ -87,78 +79,21 @@ const IndexTypeElement: FC<{
   const AddIcon = icons.add;
   const DeleteIcon = icons.delete;
 
-  const isIndexCreating: boolean = useMemo(
-    () => status === IndexState.InProgress || status === IndexState.Unissued,
-    [status]
-  );
-
-  useEffect(() => {
-    if (!data.index) {
-      return;
-    }
-    let running = true;
-
-    // define async data getter
-    const fetchStatus = async (
-      collectionName: string,
-      fieldName: string,
-      indexName: string
-    ) => {
-      // get fetch data
-      const index_descriptions = await IndexService.describeIndex(
-        collectionName
-      );
-
-      const indexDescription = index_descriptions.find(
-        i => i.field_name === fieldName
-      );
-
-      if (
-        indexDescription &&
-        indexDescription.state !== IndexState.Finished &&
-        running
-      ) {
-        // if not finished, sleep 3s
-        await sleep(3000);
-        // call self again
-        fetchStatus(collectionName, fieldName, indexName);
-      }
-
-      // update state
-      if (indexDescription) {
-        setStatus(indexDescription.state);
-      }
-    };
-    // prevent delete index trigger fetching index status
-    if (data.index.indexType !== '' && status !== IndexState.Delete) {
-      fetchStatus(
-        collectionName,
-        data.name,
-        data.index && data.index.index_name!
-      );
-    }
-
-    return () => {
-      running = false;
-    };
-  }, [collectionName, data.name, data.index && data.index.index_name]);
-
   const requestCreateIndex = async (
     params: IndexExtraParam,
     index_name: string
   ) => {
     const indexCreateParam: IndexCreateParam = {
       collection_name: collectionName,
-      field_name: data.name,
+      field_name: field.name,
       index_name,
       extra_params: params,
     };
-    await IndexService.createIndex(indexCreateParam);
+    await createIndex(indexCreateParam);
     // reset status to default empty string
-    setStatus(IndexState.Default);
     handleCloseDialog();
     openSnackBar(indexTrans('createSuccess'));
-    cb(collectionName);
+    cb && (await cb(collectionName));
   };
 
   const handleCreate = (e: MouseEvent<HTMLDivElement>) => {
@@ -171,9 +106,9 @@ const IndexTypeElement: FC<{
         component: (
           <CreateIndex
             collectionName={collectionName}
-            fieldName={data.name}
-            fieldType={data.data_type as DataTypeStringEnum}
-            dimension={Number(data.dimension)}
+            fieldName={field.name}
+            fieldType={field.data_type as DataTypeStringEnum}
+            dimension={Number(field.dimension)}
             handleCancel={handleCloseDialog}
             handleCreate={requestCreateIndex}
           />
@@ -185,14 +120,12 @@ const IndexTypeElement: FC<{
   const requestDeleteIndex = async () => {
     const indexDeleteParam: IndexManageParam = {
       collection_name: collectionName,
-      field_name: data.name,
-      index_name: data.index.index_name!,
+      field_name: field.name,
+      index_name: field.index.index_name,
     };
 
-    await IndexService.deleteIndex(indexDeleteParam);
-    // use 'delete' as special status for whether fetching index status check
-    setStatus(IndexState.Delete);
-    cb(collectionName);
+    await dropIndex(indexDeleteParam);
+    cb && (await cb(collectionName));
     handleCloseDialog();
     openSnackBar(successTrans('delete', { name: indexTrans('index') }));
   };
@@ -215,17 +148,16 @@ const IndexTypeElement: FC<{
   };
 
   const generateElement = () => {
-    // only vector type field is able to create index
     if (
-      data.is_primary_key ||
+      field.is_primary_key ||
       NONE_INDEXABLE_DATA_TYPES.indexOf(
-        data.data_type as DataTypeStringEnum
+        field.data_type as DataTypeStringEnum
       ) !== -1
     ) {
       return <div className={classes.item}>--</div>;
     }
 
-    if (!data.index) {
+    if (!field.index) {
       return (
         <div role="button" onClick={handleCreate} className={`${classes.btn}`}>
           <AddIcon classes={{ root: classes.addIcon }} />
@@ -234,23 +166,15 @@ const IndexTypeElement: FC<{
       );
     }
     // indexType example: FLAT
-    switch (data.index.indexType) {
+    switch (field.index.indexType) {
       default: {
-        /**
-         * empty string or 'delete' means fetching progress hasn't finished
-         * show loading animation for such situations
-         */
-        if (
-          status === IndexState.Default ||
-          status === IndexState.Delete ||
-          isIndexCreating
-        ) {
+        if (field.index.state === IndexState.InProgress) {
           return <StatusIcon type={ChildrenStatusType.CREATING} />;
         }
 
         const chipComp = () => (
           <Chip
-            label={data.index.indexType}
+            label={field.index.indexType}
             classes={{ root: classes.chip, label: classes.chipLabel }}
             deleteIcon={<DeleteIcon classes={{ root: 'icon' }} />}
             onDelete={handleDelete}

client/src/pages/schema/Schema.tsx

@@ -1,5 +1,5 @@
 import { makeStyles, Theme, Typography, Chip } from '@material-ui/core';
-import { useCallback, useEffect, useState } from 'react';
+import { useContext } from 'react';
 import { useParams } from 'react-router-dom';
 import AttuGrid from '@/components/grid/Grid';
 import { ColDefinitionsType } from '@/components/grid/Types';
@@ -7,9 +7,8 @@ import { useTranslation } from 'react-i18next';
 import { usePaginationHook } from '@/hooks';
 import icons from '@/components/icons/Icons';
 import { formatFieldType } from '@/utils';
-import { CollectionService } from '@/http';
+import { dataContext } from '@/context';
 import IndexTypeElement from './IndexTypeElement';
-import { FieldObject } from '@server/types';
 import { getLabelDisplayedRows } from '../search/Utils';
 
 const useStyles = makeStyles((theme: Theme) => ({
@@ -62,6 +61,7 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 const Schema = () => {
+  const { fetchCollection, collections, loading } = useContext(dataContext);
   const { collectionName = '' } = useParams<{ collectionName: string }>();
   const classes = useStyles();
   const { t: collectionTrans } = useTranslation('collection');
@@ -69,8 +69,13 @@ const Schema = () => {
   const { t: commonTrans } = useTranslation();
   const gridTrans = commonTrans('grid');
 
-  const [fields, setFields] = useState<FieldObject[]>([]);
-  const [loading, setLoading] = useState<boolean>(true);
+  // get collection
+  const collection = collections.find(
+    c => c.collection_name === collectionName
+  );
+
+  // get fields
+  const fileds = collection?.schema?.fields || [];
 
   const KeyIcon = icons.key;
 
@@ -84,28 +89,7 @@ const Schema = () => {
     order,
     orderBy,
     handleGridSort,
-  } = usePaginationHook(fields);
-
-  const fetchFields = useCallback(
-    async (collectionName: string) => {
-      try {
-        const collection = await CollectionService.getCollectionInfo(
-          collectionName
-        );
-
-        setFields(collection.schema.fields);
-        setLoading(false);
-      } catch (err) {
-        setLoading(false);
-        throw err;
-      }
-    },
-    [classes.nameWrapper, classes.paramWrapper]
-  );
-
-  useEffect(() => {
-    fetchFields(collectionName);
-  }, [collectionName, fetchFields]);
+  } = usePaginationHook(fileds);
 
   const colDefinitions: ColDefinitionsType[] = [
     {
@@ -173,9 +157,11 @@ const Schema = () => {
       formatter(f) {
         return (
           <IndexTypeElement
-            data={f}
+            field={f}
             collectionName={collectionName}
-            cb={fetchFields}
+            cb={async () => {
+              await fetchCollection(collectionName);
+            }}
           />
         );
       },

client/src/pages/schema/Types.ts

@@ -17,7 +17,7 @@ export type IndexType =
   | INDEX_TYPES_ENUM.BIN_FLAT
   | INDEX_TYPES_ENUM.MARISA_TRIE
   | INDEX_TYPES_ENUM.SORT
-  | INDEX_TYPES_ENUM.AUTO_INDEX;
+  | INDEX_TYPES_ENUM.AUTOINDEX;
 
 export interface IndexManageParam {
   collection_name: string;

client/src/utils/Validation.ts

@@ -265,16 +265,25 @@ export const getCheckResult = (param: ICheckMapParam): boolean => {
 /**
  * Check collection is loading or not
  */
-export const checkLoading = (v: CollectionObject): boolean =>
-  v.loadedPercentage !== '-1' && v.loadedPercentage !== '100';
+export const checkLoading = (v: CollectionObject): boolean => {
+  return (
+    typeof v.loadedPercentage !== 'undefined' &&
+    v.loadedPercentage !== -1 &&
+    v.loadedPercentage !== 100
+  );
+};
 
 /**
  * Check collection is index building or not.
  * @param v
  * @returns boolean
  */
-export const checkIndexBuilding = (v: any): boolean =>
-  v._indexState === ChildrenStatusType.CREATING;
+export const checkIndexBuilding = (v: CollectionObject): boolean => {
+  return Boolean(
+    v.schema &&
+      v.schema?.fields.some(field => field.index?.state === 'InProgress')
+  );
+};
 
 // get database name from url
 export const getDbValueFromUrl = (currentUrl: string) => {

server/src/app.ts

@@ -9,7 +9,6 @@ import { router as connectRouter } from './milvus';
 import { router as collectionsRouter } from './collections';
 import { router as databasesRouter } from './database';
 import { router as partitionsRouter } from './partitions';
-import { router as schemaRouter } from './schema';
 import { router as cronsRouter } from './crons';
 import { router as userRouter } from './users';
 import { router as prometheusRouter } from './prometheus';
@@ -19,7 +18,7 @@ import {
   ErrorMiddleware,
   ReqHeaderMiddleware,
 } from './middleware';
-import { CLIENT_TTL } from './utils';
+import { CLIENT_TTL, SimpleQueue } from './utils';
 import { getIp } from './utils/Network';
 import { DescribeIndexRes, MilvusClient } from './types';
 import { initWebSocket } from './socket';
@@ -35,6 +34,7 @@ export const clientCache = new LRUCache<
     address: string;
     indexCache: LRUCache<string, DescribeIndexRes>;
     database: string;
+    collectionsQueue: SimpleQueue<string>;
   }
 >({
   ttl: CLIENT_TTL,
@@ -48,7 +48,6 @@ router.use('/milvus', connectRouter);
 router.use('/databases', databasesRouter);
 router.use('/collections', collectionsRouter);
 router.use('/partitions', partitionsRouter);
-router.use('/schema', schemaRouter);
 router.use('/crons', cronsRouter);
 router.use('/users', userRouter);
 router.use('/prometheus', prometheusRouter);

server/src/collections/collections.controller.ts

@@ -11,6 +11,7 @@ import {
   QueryDto,
   RenameCollectionDto,
   DuplicateCollectionDto,
+  ManageIndexDto,
 } from './dto';
 
 export class CollectionController {
@@ -32,6 +33,15 @@ export class CollectionController {
     this.router.get('/', this.showCollections.bind(this));
     // get all collections statistics
     this.router.get('/statistics', this.getStatistics.bind(this));
+    // index
+    this.router.post(
+      '/index',
+      dtoValidationMiddleware(ManageIndexDto),
+      this.manageIndex.bind(this)
+    );
+
+    this.router.get('/index', this.describeIndex.bind(this));
+    this.router.post('/index/flush', this.clearCache.bind(this));
 
     // get collection with index info
     this.router.get('/:name', this.describeCollection.bind(this));
@@ -109,6 +119,7 @@ export class CollectionController {
     this.router.get('/:name/qsegments', this.getQSegment.bind(this));
     // compact
     this.router.put('/:name/compact', this.compact.bind(this));
+
     return this.router;
   }
 
@@ -201,7 +212,7 @@ export class CollectionController {
     try {
       const result = await this.collectionsService.getAllCollections(
         req.clientId,
-        name
+        [name]
       );
       res.send(result[0]);
     } catch (error) {
@@ -361,10 +372,15 @@ export class CollectionController {
 
   async dropAlias(req: Request, res: Response, next: NextFunction) {
     const alias = req.params?.alias;
+    const name = req.params?.name;
     try {
-      const result = await this.collectionsService.dropAlias(req.clientId, {
-        alias,
-      });
+      const result = await this.collectionsService.dropAlias(
+        req.clientId,
+        name,
+        {
+          alias,
+        }
+      );
       res.send(result);
     } catch (error) {
       next(error);
@@ -453,4 +469,49 @@ export class CollectionController {
       next(error);
     }
   }
+
+  async manageIndex(req: Request, res: Response, next: NextFunction) {
+    const { type, collection_name, index_name, extra_params, field_name } =
+      req.body;
+    try {
+      const result =
+        type.toLocaleLowerCase() === 'create'
+          ? await this.collectionsService.createIndex(req.clientId, {
+              collection_name,
+              extra_params,
+              field_name,
+              index_name,
+            })
+          : await this.collectionsService.dropIndex(req.clientId, {
+              collection_name,
+              field_name,
+              index_name,
+            });
+      res.send(result);
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  async describeIndex(req: Request, res: Response, next: NextFunction) {
+    const data = '' + req.query?.collection_name;
+    try {
+      const result = await this.collectionsService.describeIndex(req.clientId, {
+        collection_name: data,
+      });
+
+      res.send(result);
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  async clearCache(req: Request, res: Response, next: NextFunction) {
+    try {
+      const result = await this.collectionsService.clearCache(req.clientId);
+      res.send(result);
+    } catch (error) {
+      next(error);
+    }
+  }
 }

server/src/collections/collections.service.ts

@@ -22,16 +22,22 @@ import {
   CountReq,
   GetLoadStateReq,
   CollectionData,
+  CreateIndexReq,
+  DescribeIndexReq,
+  DropIndexReq,
 } from '@zilliz/milvus2-sdk-node';
 import { Parser } from '@json2csv/plainjs';
 import {
   throwErrorFromSDK,
   findKeyValue,
+  getKeyValueListFromJsonString,
   genRows,
   ROW_COUNT,
   convertFieldSchemaToFieldType,
   LOADING_STATE,
   DYNAMIC_FIELD,
+  SimpleQueue,
+  MIN_INT64,
 } from '../utils';
 import { QueryDto, ImportSampleDto, GetReplicasDto } from './dto';
 import {
@@ -42,18 +48,14 @@ import {
   DescribeCollectionRes,
   CountObject,
   StatisticsObject,
+  CollectionFullObject,
+  DescribeIndexRes,
 } from '../types';
-import { SchemaService } from '../schema/schema.service';
 import { clientCache } from '../app';
-import { MIN_INT64 } from '../utils/Const';
+import { clients } from '../socket';
+import { WS_EVENTS } from '../utils';
 
 export class CollectionsService {
-  private schemaService: SchemaService;
-
-  constructor() {
-    this.schemaService = new SchemaService();
-  }
-
   async showCollections(clientId: string, data?: ShowCollectionsReq) {
     const { milvusClient } = clientCache.get(clientId);
     const res = await milvusClient.showCollections(data);
@@ -64,8 +66,12 @@ export class CollectionsService {
   async createCollection(clientId: string, data: CreateCollectionReq) {
     const { milvusClient } = clientCache.get(clientId);
     const res = await milvusClient.createCollection(data);
+    const newCollection = (await this.getAllCollections(clientId, [
+      data.collection_name,
+    ])) as CollectionFullObject[];
+
     throwErrorFromSDK(res);
-    return res;
+    return newCollection[0];
   }
 
   async describeCollection(clientId: string, data: DescribeCollectionReq) {
@@ -75,7 +81,7 @@ export class CollectionsService {
     )) as DescribeCollectionRes;
 
     // get index info for collections
-    const indexRes = await this.schemaService.describeIndex(clientId, {
+    const indexRes = await this.describeIndex(clientId, {
       collection_name: data.collection_name,
     });
 
@@ -147,7 +153,12 @@ export class CollectionsService {
     const { milvusClient } = clientCache.get(clientId);
     const res = await milvusClient.renameCollection(data);
     throwErrorFromSDK(res);
-    return res;
+
+    const newCollection = (await this.getAllCollections(clientId, [
+      data.new_collection_name,
+    ])) as CollectionFullObject[];
+
+    return newCollection[0];
   }
 
   async dropCollection(clientId: string, data: DropCollectionReq) {
@@ -161,14 +172,24 @@ export class CollectionsService {
     const { milvusClient } = clientCache.get(clientId);
     const res = await milvusClient.loadCollection(data);
     throwErrorFromSDK(res);
-    return res;
+
+    const newCollection = (await this.getAllCollections(clientId, [
+      data.collection_name,
+    ])) as CollectionFullObject[];
+
+    return newCollection[0];
   }
 
   async releaseCollection(clientId: string, data: ReleaseLoadCollectionReq) {
     const { milvusClient } = clientCache.get(clientId);
     const res = await milvusClient.releaseCollection(data);
     throwErrorFromSDK(res);
-    return res;
+
+    const newCollection = (await this.getAllCollections(clientId, [
+      data.collection_name,
+    ])) as CollectionFullObject[];
+
+    return newCollection[0];
   }
 
   async getCollectionStatistics(
@@ -233,7 +254,12 @@ export class CollectionsService {
     const { milvusClient } = clientCache.get(clientId);
     const res = await milvusClient.createAlias(data);
     throwErrorFromSDK(res);
-    return res;
+
+    const newCollection = (await this.getAllCollections(clientId, [
+      data.collection_name,
+    ])) as CollectionFullObject[];
+
+    return newCollection[0];
   }
 
   async alterAlias(clientId: string, data: AlterAliasReq) {
@@ -243,11 +269,20 @@ export class CollectionsService {
     return res;
   }
 
-  async dropAlias(clientId: string, data: DropAliasReq) {
+  async dropAlias(
+    clientId: string,
+    collection_name: string,
+    data: DropAliasReq
+  ) {
     const { milvusClient } = clientCache.get(clientId);
     const res = await milvusClient.dropAlias(data);
     throwErrorFromSDK(res);
-    return res;
+
+    const newCollection = (await this.getAllCollections(clientId, [
+      collection_name,
+    ])) as CollectionFullObject[];
+
+    return newCollection[0];
   }
 
   async getReplicas(clientId: string, data: GetReplicasDto) {
@@ -280,7 +315,12 @@ export class CollectionsService {
     loadCollection: CollectionData,
     lazy: boolean = false
   ) {
+    const { collectionsQueue } = clientCache.get(clientId);
     if (lazy) {
+      // add to lazy queue
+      collectionsQueue.enqueue(collection.name);
+
+      // return lazy object
       return {
         id: collection.id,
         collection_name: collection.name,
@@ -327,13 +367,13 @@ export class CollectionsService {
 
     // loading info
     const loadedPercentage = !loadCollection
-      ? '-1'
-      : loadCollection.loadedPercentage;
+      ? -1
+      : Number(loadCollection.loadedPercentage);
 
     const status =
-      loadedPercentage === '-1'
+      loadedPercentage === -1
         ? LOADING_STATE.UNLOADED
-        : loadedPercentage === '100'
+        : loadedPercentage === 100
         ? LOADING_STATE.LOADED
         : LOADING_STATE.LOADING;
 
@@ -354,10 +394,19 @@ export class CollectionsService {
     };
   }
 
+  // get all collections details
   async getAllCollections(
     clientId: string,
-    collectionName?: string
+    collectionName: string[] = []
   ): Promise<CollectionObject[]> {
+    const cache = clientCache.get(clientId);
+
+    // clear collectionsQueue
+    if (collectionName.length === 0) {
+      cache.collectionsQueue.stop();
+      cache.collectionsQueue = new SimpleQueue<string>();
+    }
+
     // get all collections(name, timestamp, id)
     const allCollections = await this.showCollections(clientId);
     // get all loaded collection
@@ -372,33 +421,55 @@ export class CollectionsService {
       (a, b) => Number(b.timestamp) - Number(a.timestamp)
     );
 
-    // get single collection details
-    const targetCollections = allCollections.data.find(
-      d => d.name === collectionName
+    // get target collections details
+    const targetCollections = allCollections.data.filter(
+      d => collectionName.indexOf(d.name) !== -1
     );
-    if (targetCollections) {
-      const res = await this.getCollection(
-        clientId,
-        targetCollections,
-        loadedCollections.data.find(v => v.name === targetCollections.name),
-        false
-      );
-      return [res];
-    }
+
+    const targets =
+      targetCollections.length > 0 ? targetCollections : allCollections.data;
 
     // get all collection details
-    for (let i = 0; i < allCollections.data.length; i++) {
-      const collection = allCollections.data[i];
+    for (let i = 0; i < targets.length; i++) {
+      const collection = targets[i];
+      const loadedCollection = loadedCollections.data.find(
+        v => v.name === collection.name
+      );
+
+      const notLazy = !!loadedCollection || i < 5; // lazy is true, only load full details for the first 10 collections
+
       data.push(
         await this.getCollection(
           clientId,
           collection,
-          loadedCollections.data.find(v => v.name === collection.name),
-          false
+          loadedCollection,
+          !notLazy
         )
       );
     }
 
+    // start the queue
+    if (cache.collectionsQueue.size() > 0) {
+      cache.collectionsQueue.executeNext(async (collectionsToGet, q) => {
+        // if the queue is obseleted, return
+        if (q.isObseleted) {
+          return;
+        }
+        try {
+          // get current socket
+          const socketClient = clients.get(clientId);
+          // get collections
+          const res = await this.getAllCollections(clientId, collectionsToGet);
+
+          // emit event to current client
+          socketClient.emit(WS_EVENTS.COLLECTION_UPDATE, res);
+        } catch (e) {
+          console.log('ignore queue error');
+        }
+      }, 5);
+    }
+
+    // return data
     return data;
   }
 
@@ -552,4 +623,93 @@ export class CollectionsService {
 
     return res;
   }
+
+  async createIndex(clientId: string, data: CreateIndexReq) {
+    const { milvusClient, indexCache, database } = clientCache.get(clientId);
+    const res = await milvusClient.createIndex(data);
+    throwErrorFromSDK(res);
+    const key = `${database}/${data.collection_name}`;
+    // clear cache;
+    indexCache.delete(key);
+
+    // fetch new collections
+    const newCollection = (await this.getAllCollections(clientId, [
+      data.collection_name,
+    ])) as CollectionFullObject[];
+
+    throwErrorFromSDK(res);
+    return newCollection[0];
+  }
+
+  /**
+   * This function is used to describe an index in Milvus.
+   * It first checks if the index description is cached, if so, it returns the cached value.
+   * If not, it calls the Milvus SDK's describeIndex function to get the index description.
+   * If the index is finished building, it caches the index description for future use.
+   * If the index is not finished building, it deletes any cached value for this index.
+   * @param data - The request data for describing an index. It contains the collection name.
+   * @returns - The response from the Milvus SDK's describeIndex function or the cached index description.
+   */
+  async describeIndex(clientId: string, data: DescribeIndexReq) {
+    const { milvusClient, indexCache, database } = clientCache.get(clientId);
+
+    // Get the collection name from the request data
+    const key = `${database}/${data.collection_name}`;
+
+    // Try to get the index description from the cache
+    const value = indexCache.get(key);
+
+    // If the index description is in the cache, return it
+    if (value) {
+      return value as DescribeIndexRes;
+    } else {
+      // If the index description is not in the cache, call the Milvus SDK's describeIndex function
+      const res = (await milvusClient.describeIndex(data)) as DescribeIndexRes;
+
+      res.index_descriptions.map(index => {
+        // get indexType
+        index.indexType = (index.params.find(p => p.key === 'index_type')
+          ?.value || '') as string;
+        // get metricType
+        const metricTypePair =
+          index.params.filter(v => v.key === 'metric_type') || [];
+        index.metricType = findKeyValue(
+          metricTypePair,
+          'metric_type'
+        ) as string;
+        // get index parameter pairs
+        const paramsJSONstring = findKeyValue(index.params, 'params'); // params is a json string
+        const params =
+          (paramsJSONstring &&
+            getKeyValueListFromJsonString(paramsJSONstring as string)) ||
+          [];
+        index.indexParameterPairs = [...metricTypePair, ...params];
+      });
+
+      // Return the response from the Milvus SDK's describeIndex function
+      return res;
+    }
+  }
+
+  async dropIndex(clientId: string, data: DropIndexReq) {
+    const { milvusClient, indexCache, database } = clientCache.get(clientId);
+    const res = await milvusClient.dropIndex(data);
+    throwErrorFromSDK(res);
+
+    const key = `${database}/${data.collection_name}`;
+
+    // clear cache;
+    indexCache.delete(key);
+    // fetch new collections
+    const newCollection = (await this.getAllCollections(clientId, [
+      data.collection_name,
+    ])) as CollectionFullObject[];
+
+    return newCollection[0];
+  }
+
+  async clearCache(clientId: string) {
+    const { indexCache } = clientCache.get(clientId);
+    return indexCache.clear();
+  }
 }

server/src/collections/dto.ts

@@ -13,8 +13,9 @@ import {
   FieldType,
   ShowCollectionsType,
   DataType,
+  CreateIndexParam,
   SearchParam,
-} from '@zilliz/milvus2-sdk-node/dist/milvus';
+} from '@zilliz/milvus2-sdk-node';
 
 enum VectorTypes {
   Binary = DataType.BinaryVector,
@@ -129,3 +130,53 @@ export class DuplicateCollectionDto {
   @IsNotEmpty({ message: 'new_collection_name is empty.' })
   new_collection_name: string;
 }
+
+export enum ManageType {
+  DELETE = 'delete',
+  CREATE = 'create',
+}
+
+export class ManageIndexDto {
+  @IsEnum(ManageType, { message: 'Type allow delete and create' })
+  readonly type: ManageType;
+
+  @IsString()
+  readonly collection_name: string;
+
+  @IsString()
+  readonly field_name: string;
+
+  @IsObject()
+  @IsOptional()
+  readonly extra_params?: CreateIndexParam;
+}
+
+export class DescribeIndexDto {
+  @IsString()
+  readonly collection_name: string;
+
+  @IsString()
+  @IsOptional()
+  readonly field_name?: string;
+}
+
+export class GetIndexStateDto {
+  @IsString()
+  readonly collection_name: string;
+
+  @IsString()
+  @IsOptional()
+  readonly field_name?: string;
+}
+
+export class GetIndexProgressDto {
+  @IsString()
+  readonly collection_name: string;
+
+  @IsString()
+  readonly index_name: string;
+
+  @IsString()
+  @IsOptional()
+  readonly field_name?: string;
+}

server/src/crons/crons.controller.ts

@@ -10,7 +10,7 @@ export class CronsController {
   private cronsService: CronsService;
 
   constructor() {
-    this.schedulerRegistry = new SchedulerRegistry([]);
+    this.schedulerRegistry = new SchedulerRegistry(new Map());
     this.cronsService = new CronsService(
       collectionsService,
       this.schedulerRegistry

server/src/crons/crons.service.ts

@@ -1,7 +1,25 @@
 import { schedule, ScheduledTask } from 'node-cron';
 import { CollectionsService } from '../collections/collections.service';
-import { WS_EVENTS, WS_EVENTS_TYPE } from '../utils';
+import {
+  WS_EVENTS,
+  WS_EVENTS_TYPE,
+  checkLoading,
+  checkIndexing,
+} from '../utils';
 import { clients } from '../socket';
+import { CronJobObject } from '../types';
+interface CronJob {
+  id: string;
+  clientId: string; // milvus milvusClientId
+  task: ScheduledTask;
+  data: CronJobObject;
+}
+
+const getId = (clientId: string, data: CronJobObject) => {
+  return `${clientId}/${data.name}/${
+    data.payload.database
+  }/[${data.payload.collections.join('/')}]`;
+};
 
 export class CronsService {
   constructor(
@@ -9,68 +27,95 @@ export class CronsService {
     private schedulerRegistry: SchedulerRegistry
   ) {}
 
-  async toggleCronJobByName(
-    clientId: string,
-    data: {
-      name: string;
-      type: WS_EVENTS_TYPE;
-    }
-  ) {
+  async toggleCronJobByName(clientId: string, data: CronJobObject) {
     const { name, type } = data;
 
+    // define cronJob
+    const cronJob: CronJob = this.schedulerRegistry.getCronJob(clientId, data);
+
+    // if type is stop, stop cronJob
+    if (cronJob && type === WS_EVENTS_TYPE.STOP) {
+      return this.schedulerRegistry.deleteCronJob(clientId, data);
+    }
+
+    // switch case for different events
     switch (name) {
-      case WS_EVENTS.COLLECTION:
-        const cronJobEntity = this.schedulerRegistry.getCronJob(clientId, name);
-        if (!cronJobEntity && Number(type) === WS_EVENTS_TYPE.START) {
-          return this.getCollections(clientId, WS_EVENTS.COLLECTION);
+      // collection loading, indexing, update
+      case WS_EVENTS.COLLECTION_UPDATE:
+        if (type === WS_EVENTS_TYPE.START && !cronJob) {
+          return this.execCollectionUpdateTask(clientId, data);
         }
-        if (!cronJobEntity) {
-          return;
-        }
-        return Number(type) === WS_EVENTS_TYPE.STOP
-          ? cronJobEntity.stop()
-          : cronJobEntity.start();
+        break;
+
       default:
         throw new Error('Unsupported event type');
     }
   }
 
-  async getCollections(clientId: string, name: string) {
+  async execCollectionUpdateTask(clientId: string, data: CronJobObject) {
     const task = async () => {
+      const currentJob: CronJob = this.schedulerRegistry.getCronJob(
+        clientId,
+        data
+      );
+
+      if (!currentJob) {
+        console.log(
+          `running getCollection task, payload:`,
+          currentJob.data.payload
+        );
+        return;
+      }
       try {
-        const res = await this.collectionService.getAllCollections(clientId);
+        const collections = await this.collectionService.getAllCollections(
+          currentJob.clientId,
+          currentJob.data.payload.collections
+        );
         // get current socket
-        const socketClient = clients.get(clientId);
-        // emit event to current client
-        socketClient.emit(WS_EVENTS.COLLECTION, res);
-        return res;
+        const socketClient = clients.get(currentJob.clientId);
+        // emit event to current client, loading and indexing events are indetified as collection update
+        socketClient.emit(WS_EVENTS.COLLECTION_UPDATE, collections);
+
+        // if all collections are loaded, stop cron
+        const LoadingOrBuildingCollections = collections.filter(v => {
+          const isLoading = checkLoading(v);
+          const isBuildingIndex = checkIndexing(v);
+
+          return isLoading || isBuildingIndex;
+        });
+
+        if (LoadingOrBuildingCollections.length === 0) {
+          this.schedulerRegistry.deleteCronJob(clientId, data);
+        }
       } catch (error) {
         // When user not connect milvus, stop cron
-        const cronJobEntity = this.schedulerRegistry.getCronJob(clientId, name);
-        if (cronJobEntity) {
-          cronJobEntity.stop();
-        }
+        this.schedulerRegistry.deleteCronJob(clientId, data);
 
         throw new Error(error);
       }
     };
-    this.schedulerRegistry.setCronJobEveryFiveSecond(clientId, name, task);
+    // every 5 seconds
+    this.schedulerRegistry.setCronJob(clientId, '*/5 * * * * *', task, data);
   }
 }
 
 export class SchedulerRegistry {
-  constructor(private cronJobList: CronJob[]) {}
+  constructor(private cronJobMap: Map<string, CronJob>) {}
 
-  getCronJob(clientId: string, name: string) {
-    const target = this.cronJobList.find(
-      item => item.name === name && item.clientId === clientId
-    );
-    return target?.entity;
+  getCronJob(clientId: string, data: CronJobObject) {
+    const targetId = getId(clientId, data);
+
+    const target = this.cronJobMap.get(targetId);
+    return target;
   }
 
-  setCronJobEveryFiveSecond(clientId: string, name: string, func: () => {}) {
-    // The cron job will run every second
-    this.setCronJob(clientId, name, '*/5 * * * * *', func);
+  deleteCronJob(clientId: string, data: CronJobObject) {
+    const targetId = getId(clientId, data);
+
+    if (this.cronJobMap.has(targetId)) {
+      this.cronJobMap.get(targetId)?.task?.stop();
+      this.cronJobMap.delete(targetId);
+    }
   }
 
   // ┌────────────── second (optional)
@@ -85,27 +130,33 @@ export class SchedulerRegistry {
   // https://www.npmjs.com/package/node-cron
   setCronJob(
     clientId: string,
-    name: string,
-    scheduler: string,
-    func: () => {}
+    cronExpression: string,
+    func: () => void,
+    data: CronJobObject
   ) {
-    const target = this.cronJobList.find(
-      item => item.name === name && item.clientId === clientId
-    );
+    const target = this.getCronJob(clientId, data);
+
     if (target) {
-      target?.entity?.stop();
+      target?.task?.stop();
     } else {
-      const task = schedule(scheduler, () => {
-        console.log(`[Scheduler:${scheduler}] ${name}: running a task.`);
+      // create task id
+      const id = getId(clientId, data);
+
+      // create task
+      const task = schedule(cronExpression, () => {
+        console.log(
+          `[cronExpression:${cronExpression}] ${data.name} ${id}: running a task.`
+        );
         func();
       });
-      this.cronJobList.push({ clientId, name, entity: task });
+
+      // save task
+      this.cronJobMap.set(id, {
+        id,
+        clientId,
+        task,
+        data,
+      });
     }
   }
 }
-
-interface CronJob {
-  clientId: string; // milvus milvusClientId
-  name: string;
-  entity: ScheduledTask;
-}

server/src/milvus/milvus.service.ts

@@ -5,7 +5,7 @@ import {
   ClientConfig,
 } from '@zilliz/milvus2-sdk-node';
 import { LRUCache } from 'lru-cache';
-import { DEFAULT_MILVUS_PORT, INDEX_TTL } from '../utils';
+import { DEFAULT_MILVUS_PORT, INDEX_TTL, SimpleQueue } from '../utils';
 import { connectivityState } from '@grpc/grpc-js';
 import { DatabasesService } from '../database/databases.service';
 import { clientCache } from '../app';
@@ -99,7 +99,8 @@ export class MilvusService {
           ttl: INDEX_TTL,
           ttlAutopurge: true,
         }),
-        database: database,
+        database,
+        collectionsQueue: new SimpleQueue<string>(),
       });
 
       await this.databaseService.use(milvusClient.clientId, database);

server/src/schema/dto.ts

@@ -1,63 +0,0 @@
-import { CreateIndexParam } from "@zilliz/milvus2-sdk-node/dist/milvus/types";
-import {
-  IsString,
-  IsEnum,
-  IsOptional,
-  IsObject,
-  IsArray,
-} from "class-validator";
-
-class KeyValuePair {
-  key: string;
-  value: string;
-}
-
-export enum ManageType {
-  DELETE = "delete",
-  CREATE = "create",
-}
-
-export class ManageIndexDto {
-  @IsEnum(ManageType, { message: "Type allow delete and create" })
-  readonly type: ManageType;
-
-  @IsString()
-  readonly collection_name: string;
-
-  @IsString()
-  readonly field_name: string;
-
-  @IsObject()
-  @IsOptional()
-  readonly extra_params?: CreateIndexParam;
-}
-
-export class DescribeIndexDto {
-  @IsString()
-  readonly collection_name: string;
-
-  @IsString()
-  @IsOptional()
-  readonly field_name?: string;
-}
-
-export class GetIndexStateDto {
-  @IsString()
-  readonly collection_name: string;
-
-  @IsString()
-  @IsOptional()
-  readonly field_name?: string;
-}
-
-export class GetIndexProgressDto {
-  @IsString()
-  readonly collection_name: string;
-
-  @IsString()
-  readonly index_name: string;
-
-  @IsString()
-  @IsOptional()
-  readonly field_name?: string;
-}

server/src/schema/index.ts

@@ -1,5 +0,0 @@
-import { SchemaController } from "./schema.controller";
-const schemaManager = new SchemaController();
-const router = schemaManager.generateRoutes();
-
-export { router };

server/src/schema/schema.controller.ts

@@ -1,73 +0,0 @@
-import { NextFunction, Request, Response, Router } from 'express';
-import { dtoValidationMiddleware } from '../middleware/validation';
-import { SchemaService } from './schema.service';
-import { ManageIndexDto } from './dto';
-import { DescribeIndexRes } from '../types';
-
-export class SchemaController {
-  private router: Router;
-  private schemaService: SchemaService;
-
-  constructor() {
-    this.schemaService = new SchemaService();
-    this.router = Router();
-  }
-
-  generateRoutes() {
-    this.router.post(
-      '/index',
-      dtoValidationMiddleware(ManageIndexDto),
-      this.manageIndex.bind(this)
-    );
-
-    this.router.get('/index', this.describeIndex.bind(this));
-    this.router.post('/index/flush', this.clearCache.bind(this));
-
-    return this.router;
-  }
-
-  async manageIndex(req: Request, res: Response, next: NextFunction) {
-    const { type, collection_name, index_name, extra_params, field_name } =
-      req.body;
-    try {
-      const result =
-        type.toLocaleLowerCase() === 'create'
-          ? await this.schemaService.createIndex(req.clientId, {
-              collection_name,
-              extra_params,
-              field_name,
-              index_name,
-            })
-          : await this.schemaService.dropIndex(req.clientId, {
-              collection_name,
-              field_name,
-              index_name,
-            });
-      res.send(result);
-    } catch (error) {
-      next(error);
-    }
-  }
-
-  async describeIndex(req: Request, res: Response, next: NextFunction) {
-    const data = '' + req.query?.collection_name;
-    try {
-      const result = (await this.schemaService.describeIndex(req.clientId, {
-        collection_name: data,
-      })) as DescribeIndexRes;
-
-      res.send(result);
-    } catch (error) {
-      next(error);
-    }
-  }
-
-  async clearCache(req: Request, res: Response, next: NextFunction) {
-    try {
-      const result = await this.schemaService.clearCache(req.clientId);
-      res.send(result);
-    } catch (error) {
-      next(error);
-    }
-  }
-}

server/src/schema/schema.service.ts

@@ -1,89 +0,0 @@
-import {
-  CreateIndexReq,
-  DescribeIndexReq,
-  DropIndexReq,
-} from '@zilliz/milvus2-sdk-node';
-import { throwErrorFromSDK } from '../utils/Error';
-import { clientCache } from '../app';
-import { DescribeIndexRes } from '../types';
-import { getKeyValueListFromJsonString, findKeyValue } from '../utils';
-
-export class SchemaService {
-  async createIndex(clientId: string, data: CreateIndexReq) {
-    const { milvusClient, indexCache, database } = clientCache.get(clientId);
-    const res = await milvusClient.createIndex(data);
-    const key = `${database}/${data.collection_name}`;
-
-    // clear cache;
-    indexCache.delete(key);
-    throwErrorFromSDK(res);
-    return res;
-  }
-
-  /**
-   * This function is used to describe an index in Milvus.
-   * It first checks if the index description is cached, if so, it returns the cached value.
-   * If not, it calls the Milvus SDK's describeIndex function to get the index description.
-   * If the index is finished building, it caches the index description for future use.
-   * If the index is not finished building, it deletes any cached value for this index.
-   * @param data - The request data for describing an index. It contains the collection name.
-   * @returns - The response from the Milvus SDK's describeIndex function or the cached index description.
-   */
-  async describeIndex(clientId: string, data: DescribeIndexReq) {
-    const { milvusClient, indexCache, database } = clientCache.get(clientId);
-
-    // Get the collection name from the request data
-    const key = `${database}/${data.collection_name}`;
-
-    // Try to get the index description from the cache
-    const value = indexCache.get(key);
-
-    // If the index description is in the cache, return it
-    if (value) {
-      return value as DescribeIndexRes;
-    } else {
-      // If the index description is not in the cache, call the Milvus SDK's describeIndex function
-      const res = (await milvusClient.describeIndex(data)) as DescribeIndexRes;
-
-      res.index_descriptions.map(index => {
-        // get indexType
-        index.indexType = (index.params.find(p => p.key === 'index_type')
-          ?.value || '') as string;
-        // get metricType
-        const metricTypePair =
-          index.params.filter(v => v.key === 'metric_type') || [];
-        index.metricType = findKeyValue(
-          metricTypePair,
-          'metric_type'
-        ) as string;
-        // get index parameter pairs
-        const paramsJSONstring = findKeyValue(index.params, 'params'); // params is a json string
-        const params =
-          (paramsJSONstring &&
-            getKeyValueListFromJsonString(paramsJSONstring as string)) ||
-          [];
-        index.indexParameterPairs = [...metricTypePair, ...params];
-      });
-
-      // Return the response from the Milvus SDK's describeIndex function
-      return res;
-    }
-  }
-
-  async dropIndex(clientId: string, data: DropIndexReq) {
-    const { milvusClient, indexCache, database } = clientCache.get(clientId);
-
-    const res = await milvusClient.dropIndex(data);
-    const key = `${database}/${data.collection_name}`;
-
-    // clear cache;
-    indexCache.delete(key);
-    throwErrorFromSDK(res);
-    return res;
-  }
-
-  async clearCache(clientId: string) {
-    const { indexCache } = clientCache.get(clientId);
-    return indexCache.clear();
-  }
-}

server/src/types/collections.type.ts

@@ -9,8 +9,7 @@ import {
   QuerySegmentInfo,
   PersistentSegmentInfo,
 } from '@zilliz/milvus2-sdk-node';
-
-import { LOADING_STATE } from '../utils';
+import { WS_EVENTS, WS_EVENTS_TYPE, LOADING_STATE } from '../utils';
 
 export interface IndexObject extends IndexDescription {
   indexType: string;
@@ -51,7 +50,7 @@ export type CollectionFullObject = {
   description: string;
   autoID: boolean;
   id: string;
-  loadedPercentage: string;
+  loadedPercentage: number;
   consistency_level: string;
   replicas: ReplicaInfo[];
   status: LOADING_STATE;
@@ -87,3 +86,12 @@ export type StatisticsObject = {
 
 export type QuerySegmentObjects = QuerySegmentInfo[];
 export type PersistentSegmentObjects = PersistentSegmentInfo[];
+
+export type CronJobObject = {
+  name: WS_EVENTS;
+  type: WS_EVENTS_TYPE;
+  payload: {
+    database: string;
+    collections: string[];
+  };
+};

server/src/types/index.ts

@@ -2,6 +2,7 @@ export {
   KeyValuePair,
   ShowCollectionsType,
   MilvusClient,
+  ResStatus,
 } from '@zilliz/milvus2-sdk-node';
 
 export * from './collections.type';

server/src/utils/Const.ts

@@ -12,7 +12,7 @@ export const INDEX_TTL = 1000 * 60 * 60;
 
 export enum WS_EVENTS {
   REGISTER = 'REGISTER',
-  COLLECTION = 'COLLECTION',
+  COLLECTION_UPDATE = 'COLLECTION_UPDATE',
 }
 
 export enum WS_EVENTS_TYPE {

server/src/utils/Queue.ts

@@ -0,0 +1,63 @@
+export class SimpleQueue<T> {
+  private arr: T[] = []; // Array to store data and maintain insertion order
+  public isObseleted: boolean = false; // Flag to indicate if queue is abandoned
+
+  // Method to add an item to the queue
+  enqueue(item: T) {
+    this.arr.push(item);
+  }
+
+  // Method to remove and return the earliest inserted item from the queue
+  dequeue(): T | undefined {
+    return this.arr.shift();
+  }
+
+  // Method to get the latest inserted item in the queue
+  getLatest(): T | undefined {
+    return this.arr[this.arr.length - 1];
+  }
+
+  // Method to get the earliest inserted item in the queue
+  getEarliest(): T | undefined {
+    return this.arr[0];
+  }
+
+  // Method to get the size of the queue
+  size(): number {
+    return this.arr.length;
+  }
+
+  // Method to check if the queue is empty
+  isEmpty(): boolean {
+    return this.arr.length === 0;
+  }
+
+  // Method to clear the queue
+  stop() {
+    this.arr = [];
+    this.isObseleted = true; // Reset the abandoned flag
+  }
+
+  // Method to execute each item in the queue sequentially until the queue is empty
+  async executeNext(
+    callback: (items: T[], q: SimpleQueue<T>) => Promise<void>,
+    count: number = 1
+  ) {
+    if (this.isObseleted) {
+      return; // If abandoned flag is set, return
+    }
+
+    // items to process
+    const items: T[] = [];
+    while (!this.isEmpty() && items.length < count) {
+      const item = this.dequeue();
+      if (item !== undefined) {
+        items.push(item);
+      }
+    }
+    // execute callback
+    if (items.length > 0) {
+      await callback(items, this);
+    }
+  }
+}

server/src/utils/Shared.ts

@@ -0,0 +1,24 @@
+import { CollectionObject } from '../types';
+
+/**
+ * Check collection is loading or not
+ */
+export const checkLoading = (v: CollectionObject): boolean => {
+  return (
+    typeof v.loadedPercentage !== 'undefined' &&
+    v.loadedPercentage !== -1 &&
+    v.loadedPercentage !== 100
+  );
+};
+
+/**
+ * Check collection is index building or not.
+ * @param v
+ * @returns boolean
+ */
+export const checkIndexing = (v: CollectionObject): boolean => {
+  return Boolean(
+    v.schema &&
+      v.schema?.fields.some(field => field.index?.state === 'InProgress')
+  );
+};

server/src/utils/index.ts

@@ -53,3 +53,5 @@ export * from './Const';
 export * from './Error';
 export * from './Helper';
 export * from './Network';
+export * from './Queue';
+export * from './Shared';