change data layer (#406)

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

client/src/App.tsx

@@ -5,26 +5,26 @@ import {
   RootProvider,
   NavProvider,
   AuthProvider,
-  WebSocketProvider,
-  PrometheusProvider,
   DataProvider,
+  PrometheusProvider,
+  SystemProvider,
 } from './context';
 
 function App() {
   return (
     <AuthProvider>
       <RootProvider>
-        <DataProvider>
+        <SystemProvider>
           <PrometheusProvider>
-            <WebSocketProvider>
+            <DataProvider>
               <NavProvider>
                 <MuiPickersUtilsProvider utils={DayjsUtils}>
                   <Router></Router>
                 </MuiPickersUtilsProvider>
               </NavProvider>
-            </WebSocketProvider>
+            </DataProvider>
           </PrometheusProvider>
-        </DataProvider>
+        </SystemProvider>
       </RootProvider>
     </AuthProvider>
   );

client/src/components/layout/Header.tsx

@@ -73,7 +73,7 @@ const useStyles = makeStyles((theme: Theme) =>
 const Header: FC<HeaderType> = props => {
   const classes = useStyles();
   const { navInfo } = useContext(navContext);
-  const { database, databases, setDatabase } = useContext(dataContext);
+  const { database, databases, setDatabase, loading } = useContext(dataContext);
   const { address, username, logout } = useContext(authContext);
   const navigate = useNavigate();
 
@@ -118,10 +118,11 @@ const Header: FC<HeaderType> = props => {
               options={dbOptions}
               variant="filled"
               wrapperClass={classes.database}
+              disabled={loading}
             />
           ) : null}
 
-          <Typography variant="h4" color="textPrimary">
+          <Typography variant="h5" color="textPrimary">
             {navInfo.navTitle}
           </Typography>
         </div>

client/src/context/Data.tsx

@@ -1,101 +1,131 @@
-import { createContext, useEffect, useState, useContext } from 'react';
-import { Database, User, MilvusService } from '@/http';
-import { parseJson, getNode, getSystemConfigs } from '@/utils';
-import { LAST_TIME_DATABASE, MILVUS_NODE_TYPE } from '@/consts';
+import {
+  createContext,
+  useCallback,
+  useContext,
+  useEffect,
+  useState,
+  useRef,
+} from 'react';
+import { io, Socket } from 'socket.io-client';
 import { authContext } from '@/context';
+import { url, Collection, MilvusService, Database } from '@/http';
+import { checkIndexBuilding, checkLoading } from '@/utils';
 import { DataContextType } from './Types';
+import { WS_EVENTS, WS_EVENTS_TYPE } from '@server/utils/Const';
+import { LAST_TIME_DATABASE } from '@/consts';
 
 export const dataContext = createContext<DataContextType>({
+  loading: false,
+  collections: [],
+  setCollections: () => {},
   database: 'default',
-  databases: ['default'],
-  data: {},
   setDatabase: () => {},
+  databases: [],
   setDatabaseList: () => {},
 });
 
 const { Provider } = dataContext;
+
 export const DataProvider = (props: { children: React.ReactNode }) => {
-  const { isAuth } = useContext(authContext);
+  // local data state
+  const [collections, setCollections] = useState<Collection[]>([]);
+  const [connected, setConnected] = useState(false);
+  const [loading, setLoading] = useState(false);
   const [database, setDatabase] = useState<string>(
     window.localStorage.getItem(LAST_TIME_DATABASE) || 'default'
   );
   const [databases, setDatabases] = useState<string[]>(['default']);
-  const [data, setData] = useState<any>({});
-
-  const fetchData = async () => {
-    try {
-      // fetch all data
-      const [databases, metrics, users, roles] = await Promise.all([
-        Database.getDatabases(),
-        MilvusService.getMetrics(),
-        User.getUsers(),
-        User.getRoles(),
-      ]);
+  // auth context
+  const { isAuth, clientId } = useContext(authContext);
+  // socket ref
+  const socket = useRef<Socket | null>(null);
 
-      // parse data
-      const parsedJson = parseJson(metrics);
+  // socket callback
+  const socketCallBack = useCallback(
+    (data: any) => {
+      const collections: Collection[] = data.map((v: any) => new Collection(v));
 
-      // get query nodes
-      const queryNodes = getNode(
-        parsedJson.allNodes,
-        MILVUS_NODE_TYPE.QUERYNODE
+      const hasLoadingOrBuildingCollection = collections.some(
+        v => checkLoading(v) || checkIndexBuilding(v)
       );
 
-      // get data nodes
-      const dataNodes = getNode(parsedJson.allNodes, MILVUS_NODE_TYPE.DATANODE);
-
-      // get data nodes
-      const indexNodes = getNode(
-        parsedJson.allNodes,
-        MILVUS_NODE_TYPE.INDEXNODE
-      );
-
-      // get root coord
-      const rootCoord = getNode(
-        parsedJson.allNodes,
-        MILVUS_NODE_TYPE.ROOTCOORD
-      )[0];
-
-      // get system config
-      const systemConfig = getSystemConfigs(parsedJson.workingNodes);
-      const deployMode = rootCoord.infos.system_info.deploy_mode;
-      const systemInfo = rootCoord.infos.system_info;
-
-      const data = {
-        users: users.usernames,
-        roles: roles.results,
-        queryNodes,
-        dataNodes,
-        indexNodes,
-        rootCoord,
-        deployMode,
-        parsedJson,
-        systemConfig,
-        systemInfo,
-      };
+      setCollections(collections);
+      // If no collection is building index or loading collection
+      // stop server cron job
+      if (!hasLoadingOrBuildingCollection) {
+        MilvusService.triggerCron({
+          name: WS_EVENTS.COLLECTION,
+          type: WS_EVENTS_TYPE.STOP,
+        });
+      }
+    },
+    [database]
+  );
 
-      // store databases
-      setDatabases(databases.db_names);
-      // store other datas
-      setData(data);
+  // http fetch collection
+  const fetchCollection = async () => {
+    try {
+      setLoading(true);
+      const res = await Collection.getCollections();
+      setCollections(res);
+      setLoading(false);
     } catch (error) {
-      // do nothing
-      console.log('fetch data error', error);
+      console.error(error);
+    } finally {
+      setLoading(false);
     }
   };
 
+  const fetchDatabase = async () => {
+    const res = await Database.getDatabases();
+    setDatabases(res.db_names);
+  };
+
   useEffect(() => {
     if (isAuth) {
-      fetchData();
+      // fetch db
+      fetchDatabase();
+      // connect to socket server
+      socket.current = io(url as string);
+      // register client
+      socket.current.emit(WS_EVENTS.REGISTER, clientId);
+
+      socket.current.on('connect', function () {
+        console.log('--- ws connected ---', clientId);
+        setConnected(true);
+      });
+
+      socket.current?.on(WS_EVENTS.COLLECTION, socketCallBack);
     } else {
-      setData({});
+      socket.current?.disconnect();
+      // clear collections
+      setCollections([]);
+      // clear database
+      setDatabases(['default']);
+      // set connected to false
+      setConnected(false);
     }
   }, [isAuth]);
 
+  useEffect(() => {
+    if (connected) {
+      // clear data
+      setCollections([]);
+      // remove all listeners
+      socket.current?.offAny();
+      // listen to collection event
+      socket.current?.on(WS_EVENTS.COLLECTION, socketCallBack);
+      // get data
+      fetchCollection();
+    }
+  }, [socketCallBack, connected]);
+
   return (
     <Provider
       value={{
-        data,
+        loading,
+        collections,
+        setCollections,
         database,
         databases,
         setDatabase,
@@ -105,4 +135,4 @@ export const DataProvider = (props: { children: React.ReactNode }) => {
       {props.children}
     </Provider>
   );
-};
+};

client/src/context/System.tsx

@@ -0,0 +1,94 @@
+import { createContext, useEffect, useState, useContext } from 'react';
+import { User, MilvusService } from '@/http';
+import { parseJson, getNode, getSystemConfigs } from '@/utils';
+import { MILVUS_NODE_TYPE } from '@/consts';
+import { authContext } from '@/context';
+import { SystemContextType } from './Types';
+
+export const systemContext = createContext<SystemContextType>({
+  data: {},
+});
+
+const { Provider } = systemContext;
+export const SystemProvider = (props: { children: React.ReactNode }) => {
+  const { isAuth } = useContext(authContext);
+
+  const [data, setData] = useState<any>({});
+
+  const fetchData = async () => {
+    try {
+      // fetch all data
+      const [metrics, users, roles] = await Promise.all([
+        MilvusService.getMetrics(),
+        User.getUsers(),
+        User.getRoles(),
+      ]);
+
+      // parse data
+      const parsedJson = parseJson(metrics);
+
+      // get query nodes
+      const queryNodes = getNode(
+        parsedJson.allNodes,
+        MILVUS_NODE_TYPE.QUERYNODE
+      );
+
+      // get data nodes
+      const dataNodes = getNode(parsedJson.allNodes, MILVUS_NODE_TYPE.DATANODE);
+
+      // get data nodes
+      const indexNodes = getNode(
+        parsedJson.allNodes,
+        MILVUS_NODE_TYPE.INDEXNODE
+      );
+
+      // get root coord
+      const rootCoord = getNode(
+        parsedJson.allNodes,
+        MILVUS_NODE_TYPE.ROOTCOORD
+      )[0];
+
+      // get system config
+      const systemConfig = getSystemConfigs(parsedJson.workingNodes);
+      const deployMode = rootCoord.infos.system_info.deploy_mode;
+      const systemInfo = rootCoord.infos.system_info;
+
+      const data = {
+        users: users.usernames,
+        roles: roles.results,
+        queryNodes,
+        dataNodes,
+        indexNodes,
+        rootCoord,
+        deployMode,
+        parsedJson,
+        systemConfig,
+        systemInfo,
+      };
+
+      // store other datas
+      setData(data);
+    } catch (error) {
+      // do nothing
+      console.log('fetch data error', error);
+    }
+  };
+
+  useEffect(() => {
+    if (isAuth) {
+      fetchData();
+    } else {
+      setData({});
+    }
+  }, [isAuth]);
+
+  return (
+    <Provider
+      value={{
+        data,
+      }}
+    >
+      {props.children}
+    </Provider>
+  );
+};

client/src/context/Types.ts

@@ -67,11 +67,7 @@ export type AuthContextType = {
   setClientId: Dispatch<SetStateAction<string>>;
 };
 
-export type DataContextType = {
-  database: string;
-  databases: string[];
-  setDatabase: Dispatch<SetStateAction<string>>;
-  setDatabaseList: Dispatch<SetStateAction<string[]>>;
+export type SystemContextType = {
   data?: any;
 };
 
@@ -92,7 +88,12 @@ export type NavContextType = {
   setNavInfo: (param: NavInfo) => void;
 };
 
-export type WebSocketType = {
+export type DataContextType = {
+  loading: boolean;
   collections: Collection[];
-  setCollections: (data: Collection[]) => void;
-};
+  setCollections: Dispatch<SetStateAction<Collection[]>>;
+  database: string;
+  setDatabase: Dispatch<SetStateAction<string>>;
+  databases: string[];
+  setDatabaseList: Dispatch<SetStateAction<string[]>>;
+};

client/src/context/WebSocket.tsx

@@ -1,72 +0,0 @@
-import { createContext, useContext, useEffect, useState, useRef } from 'react';
-import { io, Socket } from 'socket.io-client';
-import { authContext } from '@/context';
-import { url, Collection, MilvusService } from '@/http';
-import { checkIndexBuilding, checkLoading } from '@/utils';
-import { WebSocketType } from './Types';
-import { WS_EVENTS, WS_EVENTS_TYPE } from '@server/utils/Const';
-
-export const webSocketContext = createContext<WebSocketType>({
-  collections: [],
-  setCollections: data => {},
-});
-
-const { Provider } = webSocketContext;
-
-export const WebSocketProvider = (props: { children: React.ReactNode }) => {
-  const [collections, setCollections] = useState<Collection[]>([]);
-  const { isAuth, clientId } = useContext(authContext);
-  const socket = useRef<Socket | null>(null);
-
-  useEffect(() => {
-    if (isAuth) {
-      // connect to socket server
-      socket.current = io(url as string);
-      // register client
-      socket.current.emit(WS_EVENTS.REGISTER, clientId);
-
-      socket.current.on('connect', function () {
-        console.log('--- ws connected ---', clientId);
-      });
-
-      /**
-       * Because of collections data may be big, so we still use ajax to fetch data.
-       * Only when collection list includes index building or loading collection,
-       * server will keep push collections data from milvus every seconds.
-       * After all collections are not loading or building index, tell server stop pulling data.
-       */
-      socket.current.on(WS_EVENTS.COLLECTION, (data: any) => {
-        const collections: Collection[] = data.map(
-          (v: any) => new Collection(v)
-        );
-
-        const hasLoadingOrBuildingCollection = collections.some(
-          v => checkLoading(v) || checkIndexBuilding(v)
-        );
-
-        setCollections(collections);
-        // If no collection is building index or loading collection
-        // stop server cron job
-        if (!hasLoadingOrBuildingCollection) {
-          MilvusService.triggerCron({
-            name: WS_EVENTS.COLLECTION,
-            type: WS_EVENTS_TYPE.STOP,
-          });
-        }
-      });
-    } else {
-      socket.current?.disconnect();
-    }
-  }, [isAuth]);
-
-  return (
-    <Provider
-      value={{
-        collections,
-        setCollections,
-      }}
-    >
-      {props.children}
-    </Provider>
-  );
-};

client/src/context/index.tsx

@@ -1,7 +1,7 @@
 export * from './Auth';
-export * from './Data';
+export * from './System';
 export * from './Navigation';
 export * from './Prometheus';
 export * from './Root';
 export * from './Types';
-export * from './WebSocket';
+export * from './Data';

client/src/pages/collections/Collections.tsx

@@ -6,8 +6,8 @@ import Highlighter from 'react-highlight-words';
 import {
   rootContext,
   authContext,
+  systemContext,
   dataContext,
-  webSocketContext,
 } from '@/context';
 import { Collection, MilvusService, MilvusIndex } from '@/http';
 import { useNavigationHook, usePaginationHook } from '@/hooks';
@@ -82,7 +82,7 @@ const Collections = () => {
   );
 
   const { setDialog, openSnackBar } = useContext(rootContext);
-  const { collections, setCollections } = useContext(webSocketContext);
+  const { collections, setCollections } = useContext(dataContext);
   const { t: collectionTrans } = useTranslation('collection');
   const { t: btnTrans } = useTranslation('btn');
   const { t: successTrans } = useTranslation('success');

client/src/pages/overview/Overview.tsx

@@ -10,15 +10,14 @@ import {
 import { Link } from 'react-router-dom';
 import { useTranslation } from 'react-i18next';
 import dayjs from 'dayjs';
-import { rootContext, webSocketContext, dataContext } from '@/context';
+import { rootContext, dataContext, systemContext } from '@/context';
 import EmptyCard from '@/components/cards/EmptyCard';
 import icons from '@/components/icons/Icons';
 import { LOADING_STATE, MILVUS_DEPLOY_MODE } from '@/consts';
-import { WS_EVENTS, WS_EVENTS_TYPE } from '@server/utils/Const';
 import { useNavigationHook } from '@/hooks';
-import { Collection, MilvusService } from '@/http';
+import { Collection } from '@/http';
 import { ALL_ROUTER_TYPES } from '@/router/Types';
-import { checkLoading, checkIndexBuilding, formatNumber } from '@/utils';
+import { formatNumber } from '@/utils';
 import CollectionCard from './collectionCard/CollectionCard';
 import StatisticsCard from './statisticsCard/StatisticsCard';
 
@@ -113,7 +112,8 @@ type statisticsType = {
 
 const Overview = () => {
   useNavigationHook(ALL_ROUTER_TYPES.OVERVIEW);
-  const { database, databases, data } = useContext(dataContext);
+  const { database, databases, collections, loading } = useContext(dataContext);
+  const { data } = useContext(systemContext);
   const classes = useStyles();
   const theme = useTheme();
   const { t: overviewTrans } = useTranslation('overview');
@@ -123,36 +123,23 @@ const Overview = () => {
     collectionCount: 0,
     totalData: 0,
   });
-  const [loading, setLoading] = useState(false);
-  const { collections, setCollections } = useContext(webSocketContext);
+  const [loadingLocal, setLoadingLocal] = useState(false);
   const { openSnackBar } = useContext(rootContext);
 
   const fetchData = useCallback(async () => {
-    setLoading(true);
-    setCollections([]);
-    const res = (await Collection.getStatistics()) as statisticsType;
-    const collections = await Collection.getCollections();
-    const hasLoadingOrBuildingCollection = collections.some(
-      v => checkLoading(v) || checkIndexBuilding(v)
-    );
-    // if some collection is building index or loading, start pulling data
-    if (hasLoadingOrBuildingCollection) {
-      MilvusService.triggerCron({
-        name: WS_EVENTS.COLLECTION,
-        type: WS_EVENTS_TYPE.START,
-      });
-    }
+    if (loading) return;
+    setLoadingLocal(true);
+    const res = (await Collection.getStatistics()) as any;
     setStatistics(res);
-    setCollections(collections);
-    setLoading(false);
-  }, [setCollections, database]);
+    setLoadingLocal(false);
+  }, [database, collections]);
 
   useEffect(() => {
     fetchData();
   }, [fetchData]);
 
   const loadCollections = collections.filter(
-    c => c.status !== LOADING_STATE.UNLOADED
+    c => c.status !== LOADING_STATE.UNLOADED || (c as any).loaded === 2
   );
 
   const onRelease = () => {
@@ -221,6 +208,8 @@ const Overview = () => {
     return `${duration.toFixed(2)} ${unit}`;
   }, [data.rootCoord]);
 
+  const _loading = loadingLocal || loading;
+
   return (
     <section className={`page-wrapper  ${classes.overviewContainer}`}>
       <section className={classes.dbWrapper}>
@@ -245,11 +234,13 @@ const Overview = () => {
           </div>
         ) : (
           <EmptyCard
-            loading={loading}
+            loading={_loading}
             wrapperClass={classes.emptyCard}
-            icon={!loading ? <CollectionIcon /> : undefined}
+            icon={!_loading ? <CollectionIcon /> : undefined}
             text={
-              loading ? overviewTrans('loading') : collectionTrans('noLoadData')
+              _loading
+                ? overviewTrans('loading')
+                : collectionTrans('noLoadData')
             }
           />
         )}