use ws to update loading

client/src/consts/Http.ts

@@ -4,3 +4,12 @@ export enum CODE_STATUS {
 }
 
 export const START_LOADING_TIME = 350;
+
+export enum WS_EVENTS {
+  COLLECTION = 'COLLECTION',
+}
+
+export enum WS_EVENTS_TYPE {
+  START,
+  STOP,
+}

client/src/context/Types.ts

@@ -1,4 +1,5 @@
 import { Dispatch, ReactElement, SetStateAction } from 'react';
+import { CollectionView } from '../pages/collections/Types';
 import { NavInfo } from '../router/Types';
 
 export type RootContextType = {
@@ -62,3 +63,8 @@ export type NavContextType = {
   navInfo: NavInfo;
   setNavInfo: (param: NavInfo) => void;
 };
+
+export type WebSocketType = {
+  collections: CollectionView[];
+  setCollections: (data: CollectionView[]) => void;
+};

client/src/context/WebSocket.tsx

@@ -1,29 +1,65 @@
 import { createContext, useEffect, useState } from 'react';
 import { io } from 'socket.io-client';
+import { WS_EVENTS, WS_EVENTS_TYPE } from '../consts/Http';
 import { CollectionHttp } from '../http/Collection';
+import { MilvusHttp } from '../http/Milvus';
+import { CollectionView } from '../pages/collections/Types';
+import { checkIndexBuilding, checkLoading } from '../utils/Validation';
+import { WebSocketType } from './Types';
 
-export const navContext = createContext<{}>({});
+export const webSokcetContext = createContext<WebSocketType>({
+  collections: [],
+  setCollections: data => {},
+});
 
-const { Provider } = navContext;
+const { Provider } = webSokcetContext;
 
 export const WebSocketProvider = (props: { children: React.ReactNode }) => {
+  const [collections, setCollections] = useState<CollectionView[]>([]);
+
   // test code for socket
   useEffect(() => {
     console.log('----in websocket-----');
     const socket = io('http://localhost:3000');
+
     socket.on('connect', function () {
-      console.log('Connected');
+      console.log('--- ws connected ---');
+    });
 
-      socket.emit('identity', 0, (res: any) => console.log(res));
+    /**
+     * Because of collections data may be big, so we still use ajax to fetch data.
+     * Only when collection list includes index building or loading collection,
+     * server will keep push collections data from milvus every seconds.
+     * After all collections are not loading or building index, tell server stop pulling data.
+     */
+    socket.on(WS_EVENTS.COLLECTION, (data: any) => {
+      const collections: CollectionHttp[] = data.map(
+        (v: any) => new CollectionHttp(v)
+      );
 
-      socket.emit('events', { test: 'events' });
+      const hasLoadingCollection = collections.find(v => checkLoading(v));
 
-      socket.emit('senddata', { test: 'senddata' });
-    });
-    socket.on('COLLECTION', (data: any) => {
-      const collections = data.map((v: any) => new CollectionHttp(v));
-      console.log('event', collections);
+      const hasIndexBuilding = collections.find(v => checkIndexBuilding(v));
+
+      setCollections(collections);
+      // If no collection is building index or loading collection
+      // stop server cron job
+      if (!hasLoadingCollection && !hasIndexBuilding) {
+        MilvusHttp.triggerCron({
+          name: WS_EVENTS.COLLECTION,
+          type: WS_EVENTS_TYPE.STOP,
+        });
+      }
     });
   }, []);
-  return <Provider value={{}}>{props.children}</Provider>;
+  return (
+    <Provider
+      value={{
+        collections,
+        setCollections,
+      }}
+    >
+      {props.children}
+    </Provider>
+  );
 };

client/src/http/Milvus.ts

@@ -1,9 +1,11 @@
+import { WS_EVENTS, WS_EVENTS_TYPE } from '../consts/Http';
 import BaseModel from './BaseModel';
 
 export class MilvusHttp extends BaseModel {
   static CONNECT_URL = '/milvus/connect';
   static CHECK_URL = '/milvus/check';
   static FLUSH_URL = '/milvus/flush';
+  static TIGGER_CRON_URL = '/crons';
 
   constructor(props: {}) {
     super(props);
@@ -26,4 +28,11 @@ export class MilvusHttp extends BaseModel {
       },
     });
   }
+
+  static triggerCron(data: { name: WS_EVENTS; type: WS_EVENTS_TYPE }) {
+    return super.update({
+      path: this.TIGGER_CRON_URL,
+      data,
+    });
+  }
 }

client/src/pages/collections/Collections.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useContext, useEffect, useState } from 'react';
+import { useCallback, useContext, useEffect, useMemo, useState } from 'react';
 import { Link } from 'react-router-dom';
 import { useNavigationHook } from '../../hooks/Navigation';
 import { ALL_ROUTER_TYPES } from '../../router/Types';
@@ -33,6 +33,9 @@ import { parseLocationSearch } from '../../utils/Format';
 import InsertContainer from '../../components/insert/Container';
 import { MilvusHttp } from '../../http/Milvus';
 import { LOADING_STATE } from '../../consts/Milvus';
+import { webSokcetContext } from '../../context/WebSocket';
+import { WS_EVENTS, WS_EVENTS_TYPE } from '../../consts/Http';
+import { checkIndexBuilding, checkLoading } from '../../utils/Validation';
 
 const useStyles = makeStyles((theme: Theme) => ({
   emptyWrapper: {
@@ -59,88 +62,94 @@ const useStyles = makeStyles((theme: Theme) => ({
 
 let timer: NodeJS.Timeout | null = null;
 // get init search value from url
-const { search = '' } = parseLocationSearch(window.location.search);
+const { urlSearch = '' } = parseLocationSearch(window.location.search);
 
 const Collections = () => {
   useNavigationHook(ALL_ROUTER_TYPES.COLLECTIONS);
   const { handleAction } = useLoadAndReleaseDialogHook({ type: 'collection' });
   const { handleInsertDialog } = useInsertDialogHook();
-  const [collections, setCollections] = useState<CollectionView[]>([]);
 
-  const [searchedCollections, setSearchedCollections] = useState<
-    CollectionView[]
-  >([]);
-  const {
-    pageSize,
-    handlePageSize,
-    currentPage,
-    handleCurrentPage,
-    total,
-    data: collectionList,
-    handleGridSort,
-    order,
-    orderBy,
-  } = usePaginationHook(searchedCollections);
-  const [loading, setLoading] = useState<boolean>(true);
+  const [search, setSearch] = useState<string>(urlSearch);
+  const [loading, setLoading] = useState<boolean>(false);
   const [selectedCollections, setSelectedCollections] = useState<
     CollectionView[]
   >([]);
 
   const { setDialog, handleCloseDialog, openSnackBar } =
     useContext(rootContext);
+  const { collections, setCollections } = useContext(webSokcetContext);
   const { t: collectionTrans } = useTranslation('collection');
   const { t: btnTrans } = useTranslation('btn');
   const { t: dialogTrans } = useTranslation('dialog');
   const { t: successTrans } = useTranslation('success');
-
   const classes = useStyles();
 
   const LoadIcon = icons.load;
   const ReleaseIcon = icons.release;
   const InfoIcon = icons.info;
 
+  const searchedCollections = useMemo(
+    () => collections.filter(collection => collection._name.includes(search)),
+    [collections, search]
+  );
+
+  const formatCollections = useMemo(() => {
+    const data = searchedCollections.map(v => {
+      // const indexStatus = statusRes.find(item => item._name === v._name);
+      Object.assign(v, {
+        nameElement: (
+          <Link to={`/collections/${v._name}`} className={classes.link}>
+            <Highlighter
+              textToHighlight={v._name}
+              searchWords={[search]}
+              highlightClassName={classes.highlight}
+            />
+          </Link>
+        ),
+        statusElement: (
+          <Status status={v._status} percentage={v._loadedPercentage} />
+        ),
+        indexCreatingElement: (
+          <StatusIcon type={v._indexState || ChildrenStatusType.FINISH} />
+        ),
+      });
+
+      return v;
+    });
+    return data;
+  }, [classes.highlight, classes.link, search, searchedCollections]);
+
+  const {
+    pageSize,
+    handlePageSize,
+    currentPage,
+    handleCurrentPage,
+    total,
+    data: collectionList,
+    handleGridSort,
+    order,
+    orderBy,
+  } = usePaginationHook(formatCollections);
+
   const fetchData = useCallback(async () => {
     try {
+      setLoading(true);
       const res = await CollectionHttp.getCollections();
-      const statusRes = await CollectionHttp.getCollectionsIndexState();
-      setLoading(false);
+      const hasLoadingCollection = res.find(v => checkLoading(v));
 
-      const collections = res.map(v => {
-        const indexStatus = statusRes.find(item => item._name === v._name);
-        Object.assign(v, {
-          nameElement: (
-            <Link to={`/collections/${v._name}`} className={classes.link}>
-              <Highlighter
-                textToHighlight={v._name}
-                searchWords={[search]}
-                highlightClassName={classes.highlight}
-              />
-            </Link>
-          ),
-          statusElement: (
-            <Status status={v._status} percentage={v._loadedPercentage} />
-          ),
-          indexCreatingElement: (
-            <StatusIcon
-              type={indexStatus?._indexState || ChildrenStatusType.FINISH}
-            />
-          ),
+      const hasIndexBuilding = res.find(v => checkIndexBuilding(v));
+      if (hasLoadingCollection || hasIndexBuilding) {
+        MilvusHttp.triggerCron({
+          name: WS_EVENTS.COLLECTION,
+          type: WS_EVENTS_TYPE.START,
         });
+      }
 
-        return v;
-      });
-
-      // filter collection if url contains search param
-      const filteredCollections = collections.filter(collection =>
-        collection._name.includes(search)
-      );
-
-      setCollections(collections);
-      setSearchedCollections(filteredCollections);
-    } catch (err) {
+      setCollections(res);
+    } finally {
       setLoading(false);
     }
-  }, [classes.link, classes.highlight]);
+  }, [setCollections]);
 
   useEffect(() => {
     fetchData();
@@ -226,32 +235,7 @@ const Collections = () => {
     if (timer) {
       clearTimeout(timer);
     }
-    // add loading manually
-    setLoading(true);
-    timer = setTimeout(() => {
-      const searchWords = [value];
-      const list = value
-        ? collections.filter(c => c._name.includes(value))
-        : collections;
-
-      const highlightList = list.map(c => {
-        Object.assign(c, {
-          nameElement: (
-            <Link to={`/collections/${c._name}`} className={classes.link}>
-              <Highlighter
-                textToHighlight={c._name}
-                searchWords={searchWords}
-                highlightClassName={classes.highlight}
-              />
-            </Link>
-          ),
-        });
-        return c;
-      });
-
-      setLoading(false);
-      setSearchedCollections(highlightList);
-    }, 300);
+    setSearch(value);
   };
 
   const toolbarConfigs: ToolBarConfig[] = [
@@ -275,7 +259,7 @@ const Collections = () => {
       onClick: () => {
         handleInsertDialog(
           <InsertContainer
-            collections={collections}
+            collections={formatCollections}
             defaultSelectedCollection={
               selectedCollections.length === 1
                 ? selectedCollections[0]._name

client/src/utils/Validation.ts

@@ -1,3 +1,4 @@
+import { ChildrenStatusType } from '../components/status/Types';
 import { MetricType, METRIC_TYPES_VALUES } from '../consts/Milvus';
 
 export type ValidType =
@@ -243,3 +244,17 @@ export const getCheckResult = (param: ICheckMapParam): boolean => {
 
   return checkMap[rule];
 };
+
+/**
+ * Check collection is loading or not
+ */
+export const checkLoading = (v: any): boolean =>
+  v._loadedPercentage !== '-1' && v._loadedPercentage !== '100';
+
+/**
+ * Check collection is index building or not.
+ * @param v
+ * @returns boolean
+ */
+export const checkIndexBuilding = (v: any): boolean =>
+  v._indexState === ChildrenStatusType.CREATING;

server/generate-csv.ts

@@ -18,7 +18,7 @@ const generateVector = (dimension) => {
   return JSON.stringify(vectors);
 };
 
-while (records.length < 3000) {
+while (records.length < 50000) {
   const value = generateVector(4);
   records.push({ vector: value });
 }

server/src/collections/collections.service.ts

@@ -113,15 +113,23 @@ export class CollectionsService {
     if (res.data.length > 0) {
       for (const item of res.data) {
         const { name } = item;
+
         const collectionInfo = await this.describeCollection({
           collection_name: name,
         });
+
         const collectionStatistics = await this.getCollectionStatistics({
           collection_name: name,
         });
+
+        const indexRes = await this.getIndexStatus({
+          collection_name: item.name,
+        });
+
         const autoID = collectionInfo.schema.fields.find(
           (v) => v.is_primary_key === true,
         )?.autoID;
+
         const loadCollection = loadedCollections.data.find(
           (v) => v.name === name,
         );
@@ -129,6 +137,7 @@ export class CollectionsService {
         const loadedPercentage = !loadCollection
           ? '-1'
           : loadCollection.loadedPercentage;
+
         data.push({
           collection_name: name,
           schema: collectionInfo.schema,
@@ -138,6 +147,7 @@ export class CollectionsService {
           id: collectionInfo.collectionID,
           loadedPercentage,
           createdTime: collectionInfo.created_utc_timestamp,
+          index_status: indexRes.state,
         });
       }
     }
@@ -192,7 +202,7 @@ export class CollectionsService {
 
   /**
    * Get all collection index status
-   * @returns {collection_name:string, index_state: IndexState}[]
+   * @returns {collection_name:string, index_status: IndexState}[]
    */
   async getCollectionsIndexStatus() {
     const data = [];
@@ -204,7 +214,7 @@ export class CollectionsService {
         });
         data.push({
           collection_name: item.name,
-          index_state: indexRes.state,
+          index_status: indexRes.state,
         });
       }
     }

server/src/crons/crons.controller.ts

@@ -8,7 +8,6 @@ export class CronsController {
 
   @Put()
   async toggleCron(@Body() data: ToggleCron) {
-    console.log(data);
     return await this.cronsService.toggleCronJobByName(data);
   }
 }

server/src/crons/crons.service.ts

@@ -19,7 +19,7 @@ export class CronsService {
       : cronJob.start();
   }
 
-  @Cron(CronExpression.EVERY_30_SECONDS, { name: WS_EVENTS.COLLECTION })
+  @Cron(CronExpression.EVERY_SECOND, { name: WS_EVENTS.COLLECTION })
   async getCollections() {
     const res = await this.collectionService.getAllCollections();
     this.eventService.server.emit(WS_EVENTS.COLLECTION, res);