Support create sparse vector index (#484)

client/src/consts/Milvus.ts

@@ -47,6 +47,9 @@ export enum INDEX_TYPES_ENUM {
   BIN_FLAT = 'BIN_FLAT',
   SORT = 'STL_SORT',
   MARISA_TRIE = 'Trie',
+  // sparse
+  SPARSE_INVERTED_INDEX = 'SPARSE_INVERTED_INDEX',
+  SPARSE_WAND = 'SPARSE_WAND',
 }
 
 export enum METRIC_TYPES_VALUES {
@@ -114,7 +117,8 @@ export type searchKeywordsType =
   | 'level'
   | 'search_list'
   | 'radius'
-  | 'range_filter';
+  | 'range_filter'
+  | 'drop_ratio_search';
 
 export type indexConfigType = {
   [x: string]: {
@@ -161,6 +165,14 @@ export const FLOAT_INDEX_CONFIG: indexConfigType = {
     create: [],
     search: ['search_list'],
   },
+  SPARSE_WAND: {
+    create: ['drop_ratio_build'],
+    search: ['drop_ratio_search'],
+  },
+  SPARSE_INVERTED_INDEX: {
+    create: ['drop_ratio_build'],
+    search: ['drop_ratio_search'],
+  },
 };
 
 export const BINARY_INDEX_CONFIG: indexConfigType = {
@@ -175,9 +187,21 @@ export const BINARY_INDEX_CONFIG: indexConfigType = {
   },
 };
 
+export const SPARSE_INDEX_CONFIG: indexConfigType = {
+  SPARSE_INVERTED_INDEX: {
+    create: ['drop_ratio_build'],
+    search: ['drop_ratio_search'],
+  },
+  SPARSE_WAND: {
+    create: ['drop_ratio_build'],
+    search: ['drop_ratio_search'],
+  },
+};
+
 export const INDEX_CONFIG: indexConfigType = {
   ...FLOAT_INDEX_CONFIG,
   ...BINARY_INDEX_CONFIG,
+  ...SPARSE_INDEX_CONFIG,
 };
 
 export const COLLECTION_NAME_REGX = /^[0-9,a-z,A-Z$_]+$/;
@@ -199,6 +223,10 @@ export const INDEX_OPTIONS_MAP = {
     label: v,
     value: v,
   })),
+  [DataTypeEnum.SparseFloatVector]: Object.keys(SPARSE_INDEX_CONFIG).map(v => ({
+    label: v,
+    value: v,
+  })),
   [DataTypeEnum.VarChar]: [
     {
       label: 'marisa-trie',
@@ -207,45 +235,6 @@ export const INDEX_OPTIONS_MAP = {
   ],
 };
 
-export const METRIC_OPTIONS_MAP = {
-  [DataTypeEnum.FloatVector]: [
-    {
-      value: METRIC_TYPES_VALUES.L2,
-      label: METRIC_TYPES_VALUES.L2,
-    },
-    {
-      value: METRIC_TYPES_VALUES.IP,
-      label: METRIC_TYPES_VALUES.IP,
-    },
-    {
-      value: METRIC_TYPES_VALUES.COSINE,
-      label: METRIC_TYPES_VALUES.COSINE,
-    },
-  ],
-  [DataTypeEnum.BinaryVector]: [
-    {
-      value: METRIC_TYPES_VALUES.SUBSTRUCTURE,
-      label: METRIC_TYPES_VALUES.SUBSTRUCTURE,
-    },
-    {
-      value: METRIC_TYPES_VALUES.SUPERSTRUCTURE,
-      label: METRIC_TYPES_VALUES.SUPERSTRUCTURE,
-    },
-    {
-      value: METRIC_TYPES_VALUES.HAMMING,
-      label: METRIC_TYPES_VALUES.HAMMING,
-    },
-    {
-      value: METRIC_TYPES_VALUES.JACCARD,
-      label: METRIC_TYPES_VALUES.JACCARD,
-    },
-    {
-      value: METRIC_TYPES_VALUES.TANIMOTO,
-      label: METRIC_TYPES_VALUES.TANIMOTO,
-    },
-  ],
-};
-
 // search params default value map
 export const DEFAULT_SEARCH_PARAM_VALUE_MAP: {
   [key in searchKeywordsType]?: number;
@@ -337,6 +326,7 @@ export enum DataTypeStringEnum {
   JSON = 'JSON',
   BinaryVector = 'BinaryVector',
   FloatVector = 'FloatVector',
+  SparseFloatVector = 'SparseFloatVector',
   Array = 'Array',
   None = 'None',
 }

client/src/pages/databases/collections/overview/Create.tsx

@@ -10,6 +10,7 @@ import {
   INDEX_TYPES_ENUM,
   DataTypeEnum,
   DataTypeStringEnum,
+  vectorTypes,
 } from '@/consts';
 import { useFormValidation } from '@/hooks';
 import { getCreateIndexJSCode } from '@/utils/code/Js';
@@ -22,6 +23,7 @@ import { IndexType, IndexExtraParam } from './Types';
 const CreateIndex = (props: {
   collectionName: string;
   fieldType: DataTypeStringEnum;
+  dataType: DataTypeEnum;
   handleCreate: (params: IndexExtraParam, index_name: string) => void;
   handleCancel: () => void;
 
@@ -36,6 +38,7 @@ const CreateIndex = (props: {
     handleCreate,
     handleCancel,
     fieldName,
+    dataType,
     dimension,
   } = props;
 
@@ -50,6 +53,8 @@ const CreateIndex = (props: {
         return INDEX_TYPES_ENUM.BIN_IVF_FLAT;
       case DataTypeStringEnum.FloatVector:
         return INDEX_TYPES_ENUM.AUTOINDEX;
+      case DataTypeStringEnum.SparseFloatVector:
+        return INDEX_TYPES_ENUM.SPARSE_WAND;
       case DataTypeStringEnum.VarChar:
         return INDEX_TYPES_ENUM.MARISA_TRIE;
       case DataTypeStringEnum.Int8:
@@ -67,7 +72,8 @@ const CreateIndex = (props: {
       case DataTypeStringEnum.BinaryVector:
         return METRIC_TYPES_VALUES.HAMMING;
       case DataTypeStringEnum.FloatVector:
-        return METRIC_TYPES_VALUES.L2;
+      case DataTypeStringEnum.SparseFloatVector:
+        return METRIC_TYPES_VALUES.IP;
       default:
         return '';
     }
@@ -103,12 +109,8 @@ const CreateIndex = (props: {
   }, [indexSetting.index_type]);
 
   const metricOptions = useMemo(() => {
-    const vectorType = [
-      DataTypeStringEnum.BinaryVector,
-      DataTypeStringEnum.FloatVector,
-    ];
-    return vectorType.includes(fieldType)
-      ? getMetricOptions(indexSetting.index_type, fieldType)
+    return vectorTypes.includes(dataType)
+      ? getMetricOptions(indexSetting.index_type, dataType)
       : [];
   }, [indexSetting.index_type, fieldType]);
 
@@ -135,6 +137,8 @@ const CreateIndex = (props: {
         return INDEX_OPTIONS_MAP[DataTypeEnum.BinaryVector];
       case DataTypeStringEnum.FloatVector:
         return INDEX_OPTIONS_MAP[DataTypeEnum.FloatVector];
+      case DataTypeStringEnum.SparseFloatVector:
+        return INDEX_OPTIONS_MAP[DataTypeEnum.SparseFloatVector];
       case DataTypeStringEnum.VarChar:
         return INDEX_OPTIONS_MAP[DataTypeEnum.VarChar];
 
@@ -144,10 +148,7 @@ const CreateIndex = (props: {
   }, [fieldType]);
 
   const checkedForm = useMemo(() => {
-    if (
-      fieldType !== DataTypeStringEnum.BinaryVector &&
-      fieldType !== DataTypeStringEnum.FloatVector
-    ) {
+    if (!vectorTypes.includes(dataType)) {
       return [];
     }
     const paramsForm: any = { metric_type: indexSetting.metric_type };
@@ -162,11 +163,7 @@ const CreateIndex = (props: {
    * create index code mode
    */
   const codeBlockData: CodeViewData[] = useMemo(() => {
-    const vectorTypes = [
-      DataTypeStringEnum.BinaryVector,
-      DataTypeStringEnum.FloatVector,
-    ];
-    const isScalarField = !vectorTypes.includes(fieldType);
+    const isScalarField = !vectorTypes.includes(dataType);
     const getCodeParams = {
       collectionName,
       fieldName,
@@ -222,6 +219,7 @@ const CreateIndex = (props: {
       candidate_pool_size: '',
       search_length: '',
       knng: '',
+      drop_ratio_build: '',
     }));
   }, [indexCreateParams, setDisabled, defaultMetricType]);
 

client/src/pages/databases/collections/overview/CreateForm.tsx

@@ -112,6 +112,13 @@ const CreateForm = (
     );
     const knng = generateNumberConfig('knng', 'knng', 5, 300);
 
+    const drop_ratio_build = generateNumberConfig(
+      'drop_ratio_build',
+      'drop_ratio_build',
+      0,
+      1
+    );
+
     if (indexParams.includes('nlist')) {
       result.push(nlist);
     }
@@ -148,6 +155,10 @@ const CreateForm = (
       result.push(knng);
     }
 
+    if (indexParams.includes('drop_ratio_build')) {
+      result.push(drop_ratio_build);
+    }
+
     return result;
   }, [updateForm, warningTrans, indexParams, formValue]);
 

client/src/pages/databases/collections/overview/IndexTypeElement.tsx

@@ -8,7 +8,11 @@ import Icons from '@/components/icons/Icons';
 import DeleteTemplate from '@/components/customDialog/DeleteDialogTemplate';
 import StatusIcon, { LoadingType } from '@/components/status/StatusIcon';
 import { IndexState } from '@/types/Milvus';
-import { NONE_INDEXABLE_DATA_TYPES, DataTypeStringEnum } from '@/consts';
+import {
+  NONE_INDEXABLE_DATA_TYPES,
+  DataTypeStringEnum,
+  DataTypeEnum,
+} from '@/consts';
 import CreateIndex from './Create';
 import { FieldObject } from '@server/types';
 import CustomButton from '@/components/customButton/CustomButton';
@@ -105,6 +109,7 @@ const IndexTypeElement: FC<{
           <CreateIndex
             collectionName={collectionName}
             fieldName={field.name}
+            dataType={field.dataType as unknown as DataTypeEnum}
             fieldType={field.data_type as DataTypeStringEnum}
             dimension={Number(field.dimension)}
             handleCancel={handleCloseDialog}

client/src/pages/databases/collections/overview/Types.ts

@@ -17,7 +17,9 @@ export type IndexType =
   | INDEX_TYPES_ENUM.BIN_FLAT
   | INDEX_TYPES_ENUM.MARISA_TRIE
   | INDEX_TYPES_ENUM.SORT
-  | INDEX_TYPES_ENUM.AUTOINDEX;
+  | INDEX_TYPES_ENUM.AUTOINDEX
+  | INDEX_TYPES_ENUM.SPARSE_INVERTED_INDEX
+  | INDEX_TYPES_ENUM.SPARSE_WAND;
 
 export interface IndexManageParam {
   collection_name: string;

client/src/utils/Form.ts

@@ -1,5 +1,9 @@
 import { Option } from '@/components/customSelector/Types';
-import { METRIC_TYPES_VALUES, DataTypeStringEnum } from '@/consts';
+import {
+  METRIC_TYPES_VALUES,
+  DataTypeStringEnum,
+  DataTypeEnum,
+} from '@/consts';
 import { IForm } from '@/hooks';
 import { IndexType } from '@/pages/databases/collections/overview/Types';
 
@@ -21,7 +25,7 @@ export const formatForm = (info: IInfo): IForm[] => {
 
 export const getMetricOptions = (
   indexType: IndexType,
-  fieldType: DataTypeStringEnum
+  fieldType: DataTypeEnum
 ): Option[] => {
   const baseFloatOptions = [
     {
@@ -53,27 +57,35 @@ export const getMetricOptions = (
     },
   ];
 
-  const type = fieldType === 'FloatVector' ? 'ALL' : indexType;
-
-  const baseOptionsMap: { [key: string]: any } = {
-    BinaryVector: {
-      BIN_FLAT: [
-        ...baseBinaryOptions,
-        {
-          value: METRIC_TYPES_VALUES.SUBSTRUCTURE,
-          label: 'SUBSTRUCTURE',
-        },
+  switch (fieldType) {
+    case DataTypeEnum.FloatVector:
+    case DataTypeEnum.SparseFloatVector:
+      return [
         {
-          value: METRIC_TYPES_VALUES.SUPERSTRUCTURE,
-          label: 'SUPERSTRUCTURE',
+          value: METRIC_TYPES_VALUES.IP,
+          label: 'IP',
         },
-      ],
-      BIN_IVF_FLAT: baseBinaryOptions,
-    },
-    FloatVector: {
-      ALL: baseFloatOptions,
-    },
-  };
-
-  return baseOptionsMap[fieldType][type];
+      ];
+    case DataTypeEnum.BinaryVector:
+      switch (indexType) {
+        case 'BIN_FLAT':
+          return [
+            ...baseBinaryOptions,
+            {
+              value: METRIC_TYPES_VALUES.SUBSTRUCTURE,
+              label: 'SUBSTRUCTURE',
+            },
+            {
+              value: METRIC_TYPES_VALUES.SUPERSTRUCTURE,
+              label: 'SUPERSTRUCTURE',
+            },
+          ];
+        case 'BIN_IVF_FLAT':
+          return baseBinaryOptions;
+        default:
+          return baseBinaryOptions;
+      }
+    default:
+      return [];
+  }
 };