Create index add sizing info

Signed-off-by: tumao <yan.wang@zilliz.com>

client/src/consts/Milvus.tsx

@@ -206,3 +206,6 @@ export enum LOADING_STATE {
   LOADING,
   UNLOADED,
 }
+
+export const DEFAULT_VECTORS = 100000;
+export const DEFAULT_SEFMENT_FILE_SIZE = 1024;

client/src/i18n/cn/common.ts

@@ -38,6 +38,20 @@ const commonTrans = {
     join: 'Join our growing social community today',
     get: 'Get insight, tips and share ideas',
   },
+
+  capacity: {
+    b: 'B',
+    kb: 'KB',
+    mb: 'MB',
+    gb: 'GB',
+    tb: 'TB',
+    pb: 'PB',
+  },
+
+  size: 'Approximate size',
+  tip: 'Use 100k vectors and 1024 segment file size as example',
+  disk: 'Disk',
+  memory: 'Memory',
 };
 
 export default commonTrans;

client/src/i18n/en/common.ts

@@ -46,6 +46,11 @@ const commonTrans = {
     tb: 'TB',
     pb: 'PB',
   },
+
+  size: 'Approximate size',
+  tip: 'Use 100k vectors and 1024 segment file size as example',
+  disk: 'Disk',
+  memory: 'Memory',
 };
 
 export default commonTrans;

client/src/pages/schema/Create.tsx

@@ -4,6 +4,8 @@ import { CodeLanguageEnum, CodeViewData } from '../../components/code/Types';
 import DialogTemplate from '../../components/customDialog/DialogTemplate';
 import CustomSwitch from '../../components/customSwitch/CustomSwitch';
 import {
+  DEFAULT_SEFMENT_FILE_SIZE,
+  DEFAULT_VECTORS,
   INDEX_CONFIG,
   INDEX_OPTIONS_MAP,
   MetricType,
@@ -14,8 +16,10 @@ import { getCreateIndexJSCode } from '../../utils/code/Js';
 import { getCreateIndexPYCode } from '../../utils/code/Py';
 import { formatForm, getMetricOptions } from '../../utils/Form';
 import { getEmbeddingType } from '../../utils/search';
+import { computMilvusRecommonds, formatSize } from '../../utils/SizingTool';
 import { DataTypeStringEnum } from '../collections/Types';
 import CreateForm from './CreateForm';
+import SizingInfo from './SizingInfo';
 import { IndexType, IndexExtraParam, INDEX_TYPES_ENUM } from './Types';
 
 const CreateIndex = (props: {
@@ -26,9 +30,17 @@ const CreateIndex = (props: {
 
   // used for code mode
   fieldName: string;
+  // used for sizing info
+  dimension: number;
 }) => {
-  const { collectionName, fieldType, handleCreate, handleCancel, fieldName } =
-    props;
+  const {
+    collectionName,
+    fieldType,
+    handleCreate,
+    handleCancel,
+    fieldName,
+    dimension,
+  } = props;
 
   const { t: indexTrans } = useTranslation('index');
   const { t: dialogTrans } = useTranslation('dialog');
@@ -108,6 +120,58 @@ const CreateIndex = (props: {
     return form;
   }, [indexSetting, indexCreateParams]);
 
+  // sizing info needed param
+  const sizingInfo = useMemo(() => {
+    const { index_type } = indexSetting;
+    const { nlist, m } = indexSetting;
+    const floatTypes = [
+      INDEX_TYPES_ENUM.IVF_FLAT,
+      INDEX_TYPES_ENUM.IVF_PQ,
+      INDEX_TYPES_ENUM.IVF_SQ8,
+      INDEX_TYPES_ENUM.IVF_SQ8_HYBRID,
+      INDEX_TYPES_ENUM.FLAT,
+    ];
+    const bytesTyps = [
+      INDEX_TYPES_ENUM.BIN_FLAT,
+      INDEX_TYPES_ENUM.BIN_IVF_FLAT,
+    ];
+    const supportedTypes = [...floatTypes, ...bytesTyps];
+    // check param validation
+    if (!supportedTypes.includes(index_type)) {
+      return null;
+    }
+
+    if (!nlist) {
+      return null;
+    }
+    if (index_type === INDEX_TYPES_ENUM.IVF_PQ && !m) {
+      return null;
+    }
+    // vector 100000, segment file size 1024 as default value
+    const milvusRecommends = computMilvusRecommonds(
+      DEFAULT_VECTORS,
+      dimension,
+      Number(nlist),
+      Number(m),
+      DEFAULT_SEFMENT_FILE_SIZE * 1024 * 1024
+    );
+
+    let memoryType = 'byteMemorySize';
+    let diskType = 'byteDiskSize';
+    if (floatTypes.includes(index_type)) {
+      memoryType = 'memorySize';
+      diskType = 'diskSize';
+    }
+
+    const memorySize = milvusRecommends[memoryType][index_type];
+    const diskSize = milvusRecommends[diskType][index_type];
+
+    return {
+      memory: formatSize(memorySize),
+      disk: formatSize(diskSize),
+    };
+  }, [dimension, indexSetting]);
+
   /**
    * create index code mode
    */
@@ -200,6 +264,8 @@ const CreateIndex = (props: {
         indexParams={indexCreateParams}
         indexTypeChange={onIndexTypeChange}
       />
+
+      <SizingInfo info={sizingInfo} />
     </DialogTemplate>
   );
 };

client/src/pages/schema/IndexTypeElement.tsx

@@ -175,6 +175,7 @@ const IndexTypeElement: FC<{
             collectionName={collectionName}
             fieldName={data._fieldName}
             fieldType={data._fieldType}
+            dimension={Number(data._dimension)}
             handleCancel={handleCloseDialog}
             handleCreate={requestCreateIndex}
           />

client/src/pages/schema/SizingInfo.tsx

@@ -0,0 +1,74 @@
+import { makeStyles, Theme, Typography } from '@material-ui/core';
+import { FC } from 'react';
+import { useTranslation } from 'react-i18next';
+import CustomToolTip from '../../components/customToolTip/CustomToolTip';
+import icons from '../../components/icons/Icons';
+import { SizingInfoParam } from './Types';
+
+const useStyles = makeStyles((theme: Theme) => ({
+  wrapper: {
+    display: 'flex',
+    alignItems: 'flex-start',
+    justifyContent: 'space-between',
+  },
+  header: {
+    display: 'flex',
+
+    '& .title': {
+      color: theme.palette.milvusGrey.dark,
+    },
+  },
+  icon: {
+    fontSize: '20px',
+    marginLeft: theme.spacing(1),
+  },
+  info: {
+    display: 'flex',
+    flexDirection: 'column',
+    alignItems: 'flex-end',
+  },
+  pair: {
+    display: 'flex',
+
+    '& .key': {
+      marginRight: theme.spacing(2),
+      color: theme.palette.milvusGrey.dark,
+    },
+
+    '& .value': {},
+  },
+}));
+
+const SizingInfo: FC<SizingInfoParam> = props => {
+  const { info } = props;
+  const { t: commonTrans } = useTranslation();
+  const InfoIcon = icons.info;
+
+  const classes = useStyles();
+
+  return (
+    info && (
+      <section className={classes.wrapper}>
+        <div className={classes.header}>
+          <Typography className="title">{commonTrans('size')}</Typography>
+          <CustomToolTip title={commonTrans('tip')} placement="top">
+            <InfoIcon classes={{ root: classes.icon }} />
+          </CustomToolTip>
+        </div>
+
+        <div className={classes.info}>
+          <div className={classes.pair}>
+            <Typography className="key">{commonTrans('memory')}</Typography>
+            <Typography className="value">{info.memory}</Typography>
+          </div>
+          <div className={classes.pair}>
+            <Typography className="key">{commonTrans('disk')}</Typography>
+            <Typography className="value">{info.disk}</Typography>
+          </div>
+        </div>
+      </section>
+    )
+  );
+};
+
+export default SizingInfo;

client/src/pages/schema/Types.ts

@@ -78,3 +78,21 @@ export interface IndexExtraParam {
   metric_type: string;
   params: string;
 }
+
+export interface SizingToolParam {
+  // type is float or bytes
+  type: string;
+  indexType: string;
+  dimension: number;
+  vectors: number;
+  segmentFileSize: number;
+  nlist?: number;
+  m?: number;
+}
+
+export interface SizingInfoParam {
+  info: {
+    memory: string;
+    disk: string;
+  } | null;
+}

client/src/utils/SizingTool.ts

@@ -0,0 +1,139 @@
+import { INDEX_TYPES_ENUM } from '../pages/schema/Types';
+
+const commonValueCalculator = (
+  vector: number,
+  dimensions: number,
+  nlistArg: number,
+  fileSize: number
+) => {
+  const vectorCount = Math.min(fileSize / (dimensions * 4), vector);
+  const segmentCount = Math.round(vector / vectorCount);
+  const nlist = Math.min(nlistArg, vectorCount / 40);
+  return {
+    vectorCount,
+    segmentCount,
+    nlist,
+  };
+};
+
+const pqCalculator = (
+  vectorCount: number,
+  segmentCount: number,
+  dimensions: number,
+  m: number,
+  nlist: number
+) => {
+  const singleDiskSize =
+    nlist * dimensions * 4 + m * vectorCount + 256 * dimensions * 4;
+  const singleMemorySize = singleDiskSize + 256 * m * nlist * 4;
+  return {
+    pq_diskSize: singleDiskSize * segmentCount,
+    pq_memorySize: singleMemorySize * segmentCount,
+  };
+};
+
+export const computMilvusRecommonds = (
+  vector: number,
+  dimensions: number,
+  nlistArg: number,
+  m: number,
+  fileSize: number
+): { [key in string]: any } => {
+  const { vectorCount, segmentCount, nlist } = commonValueCalculator(
+    vector,
+    dimensions,
+    nlistArg,
+    fileSize
+  );
+
+  const { pq_diskSize, pq_memorySize } = pqCalculator(
+    vectorCount,
+    segmentCount,
+    dimensions,
+    m,
+    nlist
+  );
+
+  const size = vector * dimensions * 4;
+  const nlistSize = dimensions * 4 * nlist;
+  const byteSize = (dimensions / 8) * vector;
+
+  const rawFileSize = {
+    [INDEX_TYPES_ENUM.FLAT]: size,
+    [INDEX_TYPES_ENUM.IVF_FLAT]: size,
+    [INDEX_TYPES_ENUM.IVF_SQ8]: size,
+    [INDEX_TYPES_ENUM.IVF_SQ8_HYBRID]: size,
+    [INDEX_TYPES_ENUM.IVF_PQ]: size,
+  };
+
+  const memorySize = {
+    [INDEX_TYPES_ENUM.FLAT]: size,
+    [INDEX_TYPES_ENUM.IVF_FLAT]: size + nlistSize * segmentCount,
+    [INDEX_TYPES_ENUM.IVF_SQ8]: size * 0.25 + nlistSize * segmentCount,
+    [INDEX_TYPES_ENUM.IVF_SQ8_HYBRID]: size * 0.25 + nlistSize * segmentCount,
+    [INDEX_TYPES_ENUM.IVF_PQ]: pq_memorySize,
+  };
+
+  const diskSize = {
+    [INDEX_TYPES_ENUM.FLAT]: size,
+    [INDEX_TYPES_ENUM.IVF_FLAT]:
+      rawFileSize[INDEX_TYPES_ENUM.IVF_FLAT] +
+      memorySize[INDEX_TYPES_ENUM.IVF_FLAT],
+    [INDEX_TYPES_ENUM.IVF_SQ8]:
+      rawFileSize[INDEX_TYPES_ENUM.IVF_SQ8] +
+      memorySize[INDEX_TYPES_ENUM.IVF_SQ8],
+    [INDEX_TYPES_ENUM.IVF_SQ8_HYBRID]:
+      rawFileSize[INDEX_TYPES_ENUM.IVF_SQ8_HYBRID] +
+      memorySize[INDEX_TYPES_ENUM.IVF_SQ8_HYBRID],
+    [INDEX_TYPES_ENUM.IVF_PQ]:
+      rawFileSize[INDEX_TYPES_ENUM.IVF_PQ] + pq_diskSize,
+  };
+
+  const byteRawFileSize = {
+    [INDEX_TYPES_ENUM.BIN_FLAT]: byteSize,
+    [INDEX_TYPES_ENUM.BIN_IVF_FLAT]: byteSize,
+  };
+
+  const byteMemorySize = {
+    [INDEX_TYPES_ENUM.BIN_FLAT]: byteSize,
+    [INDEX_TYPES_ENUM.BIN_IVF_FLAT]: dimensions * nlist + byteSize,
+  };
+
+  const byteDiskSize = {
+    [INDEX_TYPES_ENUM.BIN_FLAT]: byteSize,
+    [INDEX_TYPES_ENUM.BIN_IVF_FLAT]:
+      byteRawFileSize[INDEX_TYPES_ENUM.BIN_IVF_FLAT] +
+      byteMemorySize[INDEX_TYPES_ENUM.BIN_IVF_FLAT],
+  };
+
+  return {
+    rawFileSize,
+    memorySize,
+    diskSize,
+    byteRawFileSize,
+    byteMemorySize,
+    byteDiskSize,
+  };
+};
+
+export const formatSize = (size: number) => {
+  let sizeStatus = 1;
+  let status = 'BYTE';
+  while (sizeStatus < 4 && size > 4096) {
+    size = size / 1024;
+    sizeStatus++;
+  }
+  sizeStatus === 2
+    ? (status = 'KB')
+    : sizeStatus === 3
+    ? (status = 'MB')
+    : sizeStatus === 4
+    ? (status = 'GB')
+    : sizeStatus === 5
+    ? (status = 'TB')
+    : (status = 'KB');
+
+  size = Math.ceil(size);
+
+  return `${size} ${status}`;
+};