..

Signed-off-by: min.tian <min.tian.cn@gmail.com>

client/src/App.tsx

@@ -10,8 +10,8 @@ import { PrometheusProvider } from './context/Prometheus';
 function App() {
   return (
     <AuthProvider>
-      <PrometheusProvider>
-        <RootProvider>
+      <RootProvider>
+        <PrometheusProvider>
           <WebSocketProvider>
             <NavProvider>
               <MuiPickersUtilsProvider utils={DayjsUtils}>
@@ -19,8 +19,8 @@ function App() {
               </MuiPickersUtilsProvider>
             </NavProvider>
           </WebSocketProvider>
-        </RootProvider>
-      </PrometheusProvider>
+        </PrometheusProvider>
+      </RootProvider>
     </AuthProvider>
   );
 }

client/src/consts/Localstorage.ts

@@ -1,7 +1,13 @@
 export const SESSION = 'CLOUD_SESSION';
 export const MILVUS_ADDRESS = 'milvus-address';
 export const LAST_TIME_ADDRESS = 'last-time-address';
+
 export const LAST_TIME_WITH_PROMETHEUS = 'last-time-with-prometheus';
 export const LAST_TIME_PROMETHEUS_ADDRESS = 'last-time-prometheus-address';
 export const LAST_TIME_PROMETHEUS_INSTANCE = 'last-time-prometheus-instance';
 export const LAST_TIME_PROMETHEUS_NAMESPACE = 'last-time-prometheus-namespace';
+
+export const LAST_TIME_HEALTHY_THRESHOLD_CPU =
+  'last-time-healthy-threshold-cpu';
+export const LAST_TIME_HEALTHY_THRESHOLD_MEMORY =
+  'last-time-healthy-threshold-memory';

client/src/consts/Prometheus.tsx

@@ -8,3 +8,6 @@ export const PROMETHEUS_INSTANCE_NAME =
 
 export const PROMETHEUS_NAMESPACE =
   (window as any)?._env_?.PROMETHEUS_NAMESPACE || '';
+
+export const DEFAULT_HEALTHY_THRESHOLD_CPU = 1;
+export const DEFAULT_HEALTHY_THRESHOLD_MEMORY = 8 * 1024 * 1024 * 1024;

client/src/context/Prometheus.tsx

@@ -15,6 +15,8 @@ import {
   PROMETHEUS_NAMESPACE,
   WITH_PROMETHEUS,
 } from '../consts/Prometheus';
+import { rootContext } from './Root';
+import { useTranslation } from 'react-i18next';
 
 export const prometheusContext = createContext<PrometheusContextType>({
   withPrometheus: false,
@@ -52,6 +54,9 @@ export const PrometheusProvider = (props: { children: React.ReactNode }) => {
 
   const [isPrometheusReady, setIsPrometheusReady] = useState(false);
 
+  const { openSnackBar } = useContext(rootContext);
+  const { t: prometheusTrans } = useTranslation('prometheus');
+
   useEffect(() => {
     if (!isAuth) return;
     if (withPrometheus) {
@@ -61,25 +66,30 @@ export const PrometheusProvider = (props: { children: React.ReactNode }) => {
         prometheusAddress: prometheusAddressformat,
         prometheusInstance,
         prometheusNamespace,
-      }).then(({ isReady }: { isReady: boolean }) => {
-        console.log('prometheus is ready?', isReady);
-        if (isReady) {
-          window.localStorage.setItem(LAST_TIME_WITH_PROMETHEUS, 'true');
-          window.localStorage.setItem(
-            LAST_TIME_PROMETHEUS_ADDRESS,
-            prometheusAddress
-          );
-          window.localStorage.setItem(
-            LAST_TIME_PROMETHEUS_INSTANCE,
-            prometheusInstance
-          );
-          window.localStorage.setItem(
-            LAST_TIME_PROMETHEUS_NAMESPACE,
-            prometheusNamespace
-          );
-        }
-        setIsPrometheusReady(isReady);
-      });
+      })
+        .then(({ isReady }: { isReady: boolean }) => {
+          if (isReady) {
+            window.localStorage.setItem(LAST_TIME_WITH_PROMETHEUS, 'true');
+            window.localStorage.setItem(
+              LAST_TIME_PROMETHEUS_ADDRESS,
+              prometheusAddress
+            );
+            window.localStorage.setItem(
+              LAST_TIME_PROMETHEUS_INSTANCE,
+              prometheusInstance
+            );
+            window.localStorage.setItem(
+              LAST_TIME_PROMETHEUS_NAMESPACE,
+              prometheusNamespace
+            );
+            // openSnackBar(prometheusTrans('ready'));
+          } else openSnackBar(prometheusTrans('invalid'), 'error');
+          setIsPrometheusReady(isReady);
+        })
+        .catch(err => {
+          openSnackBar(prometheusTrans('invalid'), 'error');
+          setIsPrometheusReady(false);
+        });
     } else {
       setIsPrometheusReady(false);
     }

client/src/http/BaseModel.ts

@@ -5,6 +5,7 @@ type findParamsType = {
   method?: Method;
   path: string;
   params: { [x: string]: any };
+  timeout?: number;
 };
 
 type updateParamsType = {
@@ -41,12 +42,14 @@ export default class BaseModel {
   }
 
   static async search(data: findParamsType) {
-    const { method = 'get', params = {}, path = '' } = data;
-    const res = await http({
+    const { method = 'get', params = {}, path = '', timeout } = data;
+    const httpConfig = {
       method,
       url: path,
       params,
-    });
+    } as any;
+    if (timeout) httpConfig.timeout = timeout;
+    const res = await http(httpConfig);
     return res.data.data;
   }
 

client/src/http/Prometheus.ts

@@ -21,6 +21,7 @@ export class PrometheusHttp extends BaseModel {
     return super.search({
       path: PrometheusHttp.SET_PROMETHEUS_URL,
       params: { prometheusAddress, prometheusInstance, prometheusNamespace },
+      timeout: 1000,
     });
   }
 

client/src/i18n/cn/prometheus.ts

@@ -0,0 +1,6 @@
+const prometheusTrans = {
+  ready: 'Prometheus is ready.',
+  invalid: 'Prometheus configuration is invalid.',
+};
+
+export default prometheusTrans;

client/src/i18n/en/prometheus.ts

@@ -0,0 +1,6 @@
+const prometheusTrans = {
+  ready: 'Prometheus is ready.',
+  invalid: 'Prometheus configuration is invalid.',
+};
+
+export default prometheusTrans;

client/src/i18n/index.ts

@@ -29,6 +29,8 @@ import systemViewTransEn from './en/systemView';
 import systemViewTransCn from './cn/systemView';
 import userTransEn from './en/user';
 import userTransCn from './cn/user';
+import prometheusTransEn from './en/prometheus';
+import prometheusTransCn from './cn/prometheus';
 
 export const resources = {
   cn: {
@@ -46,6 +48,7 @@ export const resources = {
     search: searchCn,
     systemView: systemViewTransCn,
     user: userTransCn,
+    prometheus: prometheusTransCn,
   },
   en: {
     translation: commonEn,
@@ -62,6 +65,7 @@ export const resources = {
     search: searchEn,
     systemView: systemViewTransEn,
     user: userTransEn,
+    prometheus: prometheusTransEn,
   },
 };
 

client/src/pages/system/SystemView.tsx

@@ -94,7 +94,6 @@ const SystemView: any = () => {
     async function fetchData() {
       const res = await MilvusHttp.getMetrics();
       setData(parseJson(res));
-      console.log(parseJson(res))
     }
     fetchData();
   }, []);

client/src/pages/systemHealthy/HealthyIndexDetailView.tsx

@@ -2,7 +2,7 @@ import { makeStyles, Theme } from '@material-ui/core';
 import { CHART_WIDTH, LINE_CHART_SMALL_HEIGHT } from './consts';
 import HealthyIndexRow from './HealthyIndexRow';
 import LineChartSmall from './LineChartSmall';
-import { ENodeService, INodeTreeStructure } from './Types';
+import { ENodeService, INodeTreeStructure, IThreshold } from './Types';
 import KeyboardArrowRightIcon from '@mui/icons-material/KeyboardArrowRight';
 import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown';
 import { Dispatch, SetStateAction, useState } from 'react';
@@ -48,7 +48,13 @@ const getStyles = makeStyles((theme: Theme) => ({
   },
 }));
 
-const HealthyIndexTreeItem = ({ node }: { node: INodeTreeStructure }) => {
+const HealthyIndexTreeItem = ({
+  node,
+  threshold,
+}: {
+  node: INodeTreeStructure;
+  threshold: IThreshold;
+}) => {
   const classes = getStyles();
   const [open, setOpen] = useState(false);
   return (
@@ -83,6 +89,7 @@ const HealthyIndexTreeItem = ({ node }: { node: INodeTreeStructure }) => {
                 data={node.cpu || []}
                 format={(v: number) => v.toFixed(3)}
                 unit={'Core'}
+                threshold={threshold.cpu}
               />
             </div>
           </div>
@@ -93,6 +100,7 @@ const HealthyIndexTreeItem = ({ node }: { node: INodeTreeStructure }) => {
                 data={node.memory || []}
                 format={(v: number) => (v / 1024 / 1024 / 1024).toFixed(1)}
                 unit={'GB'}
+                threshold={threshold.memory}
               />
             </div>
           </div>
@@ -105,9 +113,11 @@ const HealthyIndexTreeItem = ({ node }: { node: INodeTreeStructure }) => {
 const HealthyIndexWithTree = ({
   nodeTree,
   setSelectedService,
+  threshold,
 }: {
   nodeTree: INodeTreeStructure;
   setSelectedService: Dispatch<SetStateAction<ENodeService>>;
+  threshold: IThreshold;
 }) => {
   const classes = getStyles();
   return (
@@ -125,9 +135,14 @@ const HealthyIndexWithTree = ({
           </div>
         </div>
       )}
-      {nodeTree.children.map(node => (
-        <HealthyIndexTreeItem key={node.label} node={node} />
-      ))}
+      {!!nodeTree &&
+        nodeTree.children.map(node => (
+          <HealthyIndexTreeItem
+            key={node.label}
+            node={node}
+            threshold={threshold}
+          />
+        ))}
     </div>
   );
 };
@@ -165,9 +180,11 @@ const HealthyIndexWithoutTree = ({
 const HealthyIndexDetailView = ({
   nodeTree,
   setSelectedService,
+  threshold,
 }: {
   nodeTree: INodeTreeStructure;
   setSelectedService: Dispatch<SetStateAction<ENodeService>>;
+  threshold: IThreshold;
 }) => {
   return nodeTree.service === ENodeService.milvus ? (
     <HealthyIndexWithoutTree
@@ -178,6 +195,7 @@ const HealthyIndexDetailView = ({
     <HealthyIndexWithTree
       nodeTree={nodeTree}
       setSelectedService={setSelectedService}
+      threshold={threshold}
     />
   );
 };

client/src/pages/systemHealthy/HealthyIndexLegend.tsx

@@ -40,7 +40,7 @@ const HealthyIndexLegend = () => {
   return (
     <>
       {legendData.map(legend => (
-        <div className={classes.legendItem}>
+        <div key={legend.label} className={classes.legendItem}>
           <div
             className={classes.legendIcon}
             style={{

client/src/pages/systemHealthy/HealthyIndexOverview.tsx

@@ -79,12 +79,12 @@ const HealthyIndexOverview = ({
   setThreshold,
   timeRange,
   setTimeRange,
-  setSelectedService
+  setSelectedService,
 }: {
   selectedNode: INodeTreeStructure;
   lineChartsData: ILineChartData[];
   threshold: IThreshold;
-  setThreshold: Dispatch<SetStateAction<IThreshold>>;
+  setThreshold: (threshold: IThreshold) => void;
   timeRange: ITimeRangeOption;
   setTimeRange: Dispatch<SetStateAction<ITimeRangeOption>>;
   setSelectedService: Dispatch<SetStateAction<ENodeService>>;
@@ -116,12 +116,16 @@ const HealthyIndexOverview = ({
           </div>
         </div>
       </div>
-      <HealthyIndexDetailView nodeTree={selectedNode} setSelectedService={setSelectedService} />
+      <HealthyIndexDetailView
+        nodeTree={selectedNode}
+        setSelectedService={setSelectedService}
+        threshold={threshold}
+      />
       {selectedNode.service === ENodeService.milvus && (
         <div className={classes.chartView}>
           <div className={classes.titleMain}>Search Query History</div>
           {lineChartsData.map(chartData => (
-            <div className={classes.chartItem}>
+            <div key={chartData.label} className={classes.chartItem}>
               <div className={classes.chartLabel}>{chartData.label}</div>
               <div className={classes.chart}>
                 <LineChartLarge

client/src/pages/systemHealthy/HealthyIndexRow.tsx

@@ -15,6 +15,7 @@ const HealthyIndexRow = ({ statusList }: { statusList: EHealthyStatus[] }) => {
     <svg width={CHART_WIDTH} height={HEALTHY_INDEX_ROW_HEIGHT}>
       {statusList.map((status, i) => (
         <rect
+          key={`status-${i}`}
           x={i * stautsItemWidth + statusBlockGap / 2}
           y={0}
           rx={1}

client/src/pages/systemHealthy/LineChartSmall.tsx

@@ -1,6 +1,7 @@
 import * as d3 from 'd3';
 import {
   CHART_WIDTH,
+  HEALTHY_STATUS_COLORS,
   LINE_CHART_LARGE_HEIGHT,
   LINE_CHART_SMALL_HEIGHT,
   LINE_COLOR,
@@ -8,15 +9,18 @@ import {
   LINE_LABEL_Y_PADDING,
   LINE_SMALL_LABEL_FONT_SIZE,
 } from './consts';
+import { EHealthyStatus } from './Types';
 
 const LineChartSmall = ({
   data,
   format = d => d,
   unit = '',
+  threshold,
 }: {
   data: number[];
   format?: (d: any) => string;
   unit?: string;
+  threshold: number;
 }) => {
   const length = data.length;
   const width = CHART_WIDTH;
@@ -93,10 +97,22 @@ const LineChartSmall = ({
         )}
       </g>
       <g className="line">
+        {maxData >= threshold && (
+          <line
+            x1={xScale(0.5)}
+            y1={yScale(threshold)}
+            x2={xScale(data.length - 0.5)}
+            y2={yScale(threshold)}
+            stroke={HEALTHY_STATUS_COLORS[EHealthyStatus.warning]}
+            strokeWidth={3}
+            strokeLinecap="round"
+            strokeDasharray={"6 8"}
+          />
+        )}
         <path
           d={line(nodes) as any}
           fill="none"
-          stroke={`${LINE_COLOR}`}
+          stroke={LINE_COLOR}
           strokeWidth={3}
           opacity={0.8}
           strokeLinecap="round"

client/src/pages/systemHealthy/SystemHealthyView.tsx

@@ -23,6 +23,14 @@ import HealthyIndexOverview from './HealthyIndexOverview';
 import HealthyIndexDetailView from './HealthyIndexDetailView';
 import { KeyboardArrowDown } from '@material-ui/icons';
 import { timeRangeOptions } from './consts';
+import {
+  LAST_TIME_HEALTHY_THRESHOLD_CPU,
+  LAST_TIME_HEALTHY_THRESHOLD_MEMORY,
+} from '../../consts/Localstorage';
+import {
+  DEFAULT_HEALTHY_THRESHOLD_CPU,
+  DEFAULT_HEALTHY_THRESHOLD_MEMORY,
+} from '../../consts/Prometheus';
 // import data from "./data.json";
 
 const getStyles = makeStyles((theme: Theme) => ({
@@ -86,11 +94,28 @@ const SystemHealthyView = () => {
   const [timeRange, setTimeRange] = useState<ITimeRangeOption>(
     timeRangeOptions[2]
   );
-  const defaultThreshold = {
-    cpu: 1,
-    memory: 8 * 1024 * 1024 * 1024,
+
+  const [threshold, setThreshold] = useState<IThreshold>({
+    cpu: +(
+      window.localStorage.getItem(LAST_TIME_HEALTHY_THRESHOLD_CPU) ||
+      DEFAULT_HEALTHY_THRESHOLD_CPU
+    ),
+    memory: +(
+      window.localStorage.getItem(LAST_TIME_HEALTHY_THRESHOLD_MEMORY) ||
+      DEFAULT_HEALTHY_THRESHOLD_MEMORY
+    ),
+  });
+  const changeThreshold = (threshold: IThreshold) => {
+    window.localStorage.setItem(
+      LAST_TIME_HEALTHY_THRESHOLD_CPU,
+      `${threshold.cpu}`
+    );
+    window.localStorage.setItem(
+      LAST_TIME_HEALTHY_THRESHOLD_MEMORY,
+      `${threshold.memory}`
+    );
+    setThreshold(threshold);
   };
-  const [threshold, setThreshold] = useState<IThreshold>(defaultThreshold);
   const [prometheusData, setPrometheusData] = useState<IPrometheusAllData>();
   const nodeTree = useMemo<INodeTreeStructure>(
     () =>
@@ -172,7 +197,7 @@ const SystemHealthyView = () => {
           selectedNode={selectedNode}
           lineChartsData={lineChartsData}
           threshold={threshold}
-          setThreshold={setThreshold}
+          setThreshold={changeThreshold}
           timeRange={timeRange}
           setTimeRange={setTimeRange}
           setSelectedService={setSelectedService}

client/src/pages/systemHealthy/ThresholdSetting.tsx

@@ -54,7 +54,7 @@ function ThresholdSettingDialog({
   open: boolean;
   onClose: () => void;
   threshold: IThreshold;
-  setThreshold: Dispatch<SetStateAction<IThreshold>>;
+  setThreshold: (threshold: IThreshold) => void;
 }) {
   const classes = getStyles();
   const handleClose = () => {
@@ -143,7 +143,7 @@ const ThresholdSetting = ({
   setThreshold,
 }: {
   threshold: IThreshold;
-  setThreshold: Dispatch<SetStateAction<IThreshold>>;
+  setThreshold: (threshold: IThreshold) => void;
 }) => {
   const [open, setOpen] = useState(false);
 
@@ -161,7 +161,7 @@ const ThresholdSetting = ({
         onClick={handleClickOpen}
         sx={{
           cursor: 'pointer',
-          opacity: 0.8
+          opacity: 0.8,
         }}
       />
       <ThresholdSettingDialog

client/src/pages/systemHealthy/TimeRangeTabs.tsx

@@ -1,5 +1,5 @@
 import { makeStyles, Theme } from '@material-ui/core';
-import { Dispatch, SetStateAction } from 'react';
+import { Dispatch, Fragment, SetStateAction } from 'react';
 import { timeRangeOptions } from './consts';
 import { ITimeRangeOption } from './Types';
 import clsx from 'clsx';
@@ -39,10 +39,9 @@ const TimeRangeTabs = ({
   return (
     <div className={classes.root}>
       {timeRangeOptions.map((timeRangeOption, i: number) => (
-        <>
-          {i > 0 && <div className={classes.divider}> / </div>}
+        <Fragment key={timeRangeOption.label}>
+          {i > 0 && <div className={classes.divider}>{'/'}</div>}
           <div
-            key={timeRangeOption.label}
             className={clsx(
               classes.label,
               timeRangeOption.value === timeRange.value && classes.active
@@ -51,7 +50,7 @@ const TimeRangeTabs = ({
           >
             {timeRangeOption.label}
           </div>
-        </>
+        </Fragment>
       ))}
     </div>
   );

client/src/pages/systemHealthy/Topology.tsx

@@ -110,7 +110,7 @@ const Topology = ({
           const subChildPos = subChildrenPos[i];
 
           return (
-            <>
+            <g key={node.label}>
               {node.children.length > 0 && (
                 <g
                   className={classes.node}
@@ -216,7 +216,7 @@ const Topology = ({
                     : `${rootNode.service}`}
                 </text>
               </g>
-            </>
+            </g>
           );
         })}
       </svg>

client/src/pages/systemHealthy/dataHandler.ts

@@ -16,26 +16,43 @@ export const getInternalNode = (
   threshold: IThreshold
 ) => {
   const length = prometheusNodes[0].cpu.length;
-  const nodes = prometheusNodes.map(node => {
-    const healthyStatus = d3.range(length).map((_, i: number) => {
-      const cpu = node.cpu[i];
-      const memory = node.memory[i];
-      if (cpu === -1) return EHealthyStatus.noData;
-      if (cpu === -2) return EHealthyStatus.failed;
-      return cpu >= threshold.cpu || memory >= threshold.memory
-        ? EHealthyStatus.warning
-        : EHealthyStatus.healthy;
+  const nodes = prometheusNodes
+    .map(node => {
+      const healthyStatus = d3.range(length).map((_, i: number) => {
+        const cpu = node.cpu[i];
+        const memory = node.memory[i];
+        if (cpu === -1) return EHealthyStatus.noData;
+        if (cpu === -2) return EHealthyStatus.failed;
+        return cpu >= threshold.cpu || memory >= threshold.memory
+          ? EHealthyStatus.warning
+          : EHealthyStatus.healthy;
+      });
+      return {
+        service: service,
+        type: node.type === 'coord' ? ENodeType.coord : ENodeType.node,
+        label: node.pod,
+        healthyStatus,
+        cpu: node.cpu,
+        memory: node.memory,
+        children: [],
+      };
+    })
+    .sort((a, b) => {
+      const failedCountA = a.healthyStatus.filter(
+        s => s === EHealthyStatus.failed
+      ).length;
+      const failedCountB = b.healthyStatus.filter(
+        s => s === EHealthyStatus.failed
+      ).length;
+      if (failedCountA !== failedCountB) return failedCountB - failedCountA;
+      const warningCountA = a.healthyStatus.filter(
+        s => s === EHealthyStatus.warning
+      ).length;
+      const warningCountB = b.healthyStatus.filter(
+        s => s === EHealthyStatus.warning
+      ).length;
+      return warningCountB - warningCountA;
     });
-    return {
-      service: service,
-      type: node.type === 'coord' ? ENodeType.coord : ENodeType.node,
-      label: node.pod,
-      healthyStatus,
-      cpu: node.cpu,
-      memory: node.memory,
-      children: [],
-    };
-  });
   const overviewHealthyStatus = d3.range(length).map((_, i: number) => {
     if (nodes.find(node => node.healthyStatus[i] === EHealthyStatus.failed))
       return EHealthyStatus.failed;

server/src/prometheus/prometheus.service.ts

@@ -34,6 +34,10 @@ const sqLatencyMetric = 'milvus_proxy_sq_latency_bucket';
 const cpuMetric = 'process_cpu_seconds_total';
 const memoryMetric = 'process_resident_memory_bytes';
 
+const http = axios.create({
+  timeout: 1000,
+});
+
 export class PrometheusService {
   static address: string = '';
   static instance: string = '';
@@ -65,7 +69,11 @@ export class PrometheusService {
     prometheusInstance: string;
     prometheusNamespace: string;
   }) {
-    PrometheusService.isReady = await this.checkPrometheus(prometheusAddress);
+    PrometheusService.isReady = await this.checkPrometheus(
+      prometheusAddress,
+      prometheusInstance,
+      prometheusNamespace
+    );
     if (PrometheusService.isReady) {
       PrometheusService.address = prometheusAddress;
       PrometheusService.instance = prometheusInstance;
@@ -77,15 +85,34 @@ export class PrometheusService {
     };
   }
 
-  async checkPrometheus(prometheusAddress: string) {
-    const result = await axios
+  async checkPrometheus(
+    prometheusAddress: string,
+    prometheusInstance: string,
+    prometheusNamespace: string
+  ) {
+    const addressValid = await http
       .get(`http://${prometheusAddress}/-/ready`)
       .then(res => res?.status === 200)
       .catch(err => {
         console.log(err);
         return false;
       });
-    return result;
+    if (addressValid) {
+      const url =
+        `http://${prometheusAddress}/api/v1/query` +
+        `?query=milvus_num_node{` +
+        `app_kubernetes_io_instance="${prometheusInstance}",` +
+        `namespace="${prometheusNamespace}"}`;
+      const instanceValid = await http
+        .get(url)
+        .then(res => res?.data?.data?.result?.length > 0)
+        .catch(err => {
+          console.log(err);
+          return false;
+        });
+      if (instanceValid) return true;
+    }
+    return false;
   }
 
   async queryRange(expr: string, start: number, end: number, step: number) {
@@ -96,7 +123,7 @@ export class PrometheusService {
       `&start=${new Date(+start).toISOString()}` +
       `&end=${new Date(+end).toISOString()}` +
       `&step=${step / 1000}s`;
-    const result = await axios
+    const result = await http
       .get(url)
       .then(res => res.data)
       .catch(err => {