support functions (#714)

* reform create collection dialog

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* finish create bm25 varchar field

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* supprt insert data

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* fix create collection dialog

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* support search part1

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* optimize create index dialog

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* fix dialog

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* fix input box

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* fix

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* update schema page

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* update schema UI

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* fix create collection dialog

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

* make load collection more clear

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

---------

Signed-off-by: ryjiang <jiangruiyi@gmail.com>

client/src/components/status/Types.ts

@@ -8,6 +8,7 @@ export type StatusActionType = {
   action?: Function;
   onIndexCreate?: Function;
   showExtraAction?: boolean;
+  showLoadButton?: boolean;
   collection: CollectionObject;
   createIndexElement?: React.ReactNode;
 };

client/src/consts/Milvus.ts

@@ -7,29 +7,34 @@ export const MILVUS_DATABASE =
 export const DYNAMIC_FIELD = `$meta`;
 
 export enum DataTypeEnum {
+  None = 0,
   Bool = 1,
   Int8 = 2,
   Int16 = 3,
   Int32 = 4,
   Int64 = 5,
+
   Float = 10,
   Double = 11,
-  String = 20,
-  VarChar = 21,
+
+  // String = 20,
+  VarChar = 21, // variable-length strings with a specified maximum length
+  Array = 22,
   JSON = 23,
+
   BinaryVector = 100,
   FloatVector = 101,
   Float16Vector = 102,
-  SparseFloatVector = 104,
   BFloat16Vector = 103,
-  Array = 22,
+  SparseFloatVector = 104,
+  VarCharBM25 = 1000,
 }
 
 export const VectorTypes = [
-  DataTypeEnum.BinaryVector,
   DataTypeEnum.FloatVector,
-  DataTypeEnum.BFloat16Vector,
+  DataTypeEnum.BinaryVector,
   DataTypeEnum.Float16Vector,
+  DataTypeEnum.BFloat16Vector,
   DataTypeEnum.SparseFloatVector,
 ];
 
@@ -64,6 +69,7 @@ export enum METRIC_TYPES_VALUES {
   TANIMOTO = 'TANIMOTO',
   SUBSTRUCTURE = 'SUBSTRUCTURE',
   SUPERSTRUCTURE = 'SUPERSTRUCTURE',
+  BM25 = 'BM25',
 }
 
 export const METRIC_TYPES = [
@@ -99,6 +105,10 @@ export const METRIC_TYPES = [
     value: METRIC_TYPES_VALUES.TANIMOTO,
     label: 'TANIMOTO',
   },
+  {
+    value: METRIC_TYPES_VALUES.BM25,
+    label: 'BM25',
+  },
 ];
 
 export type MetricType =
@@ -392,6 +402,13 @@ export enum DataTypeStringEnum {
   Array = 'Array',
   None = 'None',
 }
+export const VectorTypesString: DataTypeStringEnum[] = [
+  DataTypeStringEnum.BinaryVector,
+  DataTypeStringEnum.FloatVector,
+  DataTypeStringEnum.BFloat16Vector,
+  DataTypeStringEnum.Float16Vector,
+  DataTypeStringEnum.SparseFloatVector,
+];
 
 export const NONE_INDEXABLE_DATA_TYPES = [DataTypeStringEnum.JSON];
 
@@ -454,3 +471,8 @@ export const databaseDefaults: Property[] = [
   { key: 'database.max.collections', value: '', desc: '', type: 'number' },
   { key: 'database.force.deny.writing', value: '', desc: '', type: 'boolean' },
 ];
+
+export enum FunctionType {
+  Unknown = 0,
+  BM25 = 1,
+}

client/src/i18n/cn/button.ts

@@ -25,7 +25,7 @@ const btnTrans = {
   importSampleData: '插入样本数据',
   loading: '加载中...',
   importing: '导入中...',
-  example: '生成随机向量',
+  example: '生成随机数据',
   rename: '重命名',
   duplicate: '复制',
   export: '导出',
@@ -36,6 +36,8 @@ const btnTrans = {
   star: '给我一颗小星星',
   applyFilter: '应用过滤器',
   createIndex: '创建索引',
+  createVectorIndex: '创建向量索引',
+  createScalarIndex: '创建标量索引',
   edit: '编辑',
   explore: '探索',
   close: '关闭',

client/src/i18n/cn/collection.ts

@@ -22,6 +22,9 @@ const collectionTrans = {
   createdTime: '创建时间',
   maxLength: '最大长度',
   dynamicSchema: '动态schema',
+  function: 'Function',
+  functionInput: 'Function输入',
+  functionOutput: 'Function输出',
 
   // table tooltip
   aliasInfo: '别名可以在向量搜索中用作Collection名称。',
@@ -34,7 +37,8 @@ const collectionTrans = {
 
   // create dialog
   createTitle: '创建Collection',
-  general: '一般信息',
+  idAndVectorFields: 'ID、向量或可用 BM25 算法处理的文本字段',
+  scalarFields: '标量字段',
   schema: 'schema',
   consistency: '一致性',
   consistencyLevel: '一致性级别',

client/src/i18n/cn/dialog.ts

@@ -14,7 +14,7 @@ const dialogTrans = {
   loadTitle: `加载 {{type}}`,
   editEntityTitle: `编辑 Entity`,
   modifyReplicaTitle: `修改 {{type}} 的副本`,
-  editAnalyzerTitle: `编辑 {{type}} 分析器`,
+  editAnalyzerTitle: `编辑分析器`,
 
   loadContent: `您正在尝试加载带有数据的 {{type}}。只有已加载的 {{type}} 可以被搜索。`,
   releaseContent: `您正在尝试发布带有数据的 {{type}}。请注意，数据将不再可用于搜索。`,

client/src/i18n/cn/search.ts

@@ -2,7 +2,6 @@ const searchTrans = {
   firstTip: '2. 输入搜索向量 {{dimensionTip}}',
   secondTip: '1. 选择Collection和字段',
   thirdTip: '搜索参数 {{metricType}}',
-  vectorPlaceholder: '请在此输入您的向量值，例如 [1, 2, 3, 4]',
   collection: '已加载的Collection',
   noCollection: '没有已加载的Collection',
   field: '向量字段',
@@ -29,6 +28,7 @@ const searchTrans = {
   consistency: '一致性',
   graphNodeHoverTip: '双击以查看更多',
   inputVectorPlaceHolder: '向量或实体ID',
+  textPlaceHolder: '请在此输入您的文本',
   partitionFilter: '分区过滤',
   loading: '加载中...',
 };

client/src/i18n/en/button.ts

@@ -25,7 +25,7 @@ const btnTrans = {
   importSampleData: 'Insert Sample Data',
   loading: 'Loading...',
   importing: 'Importing...',
-  example: 'Generate Random Vector',
+  example: 'Generate Random Data',
   rename: 'Rename',
   duplicate: 'Duplicate',
   export: 'Export',
@@ -36,6 +36,8 @@ const btnTrans = {
   star: 'Give me a Star',
   applyFilter: 'Apply Filters',
   createIndex: 'Create Index',
+  createVectorIndex: 'Vector Index',
+  createScalarIndex: 'Scalar Index',
   edit: 'Edit',
   explore: 'Explore',
   close: 'Close',

client/src/i18n/en/collection.ts

@@ -22,6 +22,9 @@ const collectionTrans = {
   createdTime: 'Created Time',
   maxLength: 'Max Length',
   dynamicSchema: 'Dynamic Schema',
+  function: 'Function',
+  functionInput: 'Input',
+  functionOutput: 'Output',
 
   // table tooltip
   aliasInfo: 'Alias can be used as collection name in vector search.',
@@ -35,7 +38,8 @@ const collectionTrans = {
 
   // create dialog
   createTitle: 'Create Collection',
-  general: 'General information',
+  idAndVectorFields: 'ID, Vector, or VarChar Fields for BM25 Processing',
+  scalarFields: 'Scalar Fields',
   schema: 'Schema',
   consistency: 'Consistency',
   consistencyLevel: 'Consistency Level',

client/src/i18n/en/dialog.ts

@@ -11,7 +11,7 @@ const dialogTrans = {
   flush: `Flush data for {{type}}`,
   loadTitle: `Load {{type}}`,
   editEntityTitle: `Edit Entity(JSON)`,
-  editAnalyzerTitle: `Edit analyzer for {{type}}`,
+  editAnalyzerTitle: `Edit Analyzer`,
   modifyReplicaTitle: `Modify replica for {{type}}`,
 
   loadContent: `You are trying to load a {{type}} with data. Only loaded {{type}} can be searched.`,

client/src/i18n/en/search.ts

@@ -2,7 +2,6 @@ const searchTrans = {
   firstTip: '2. Enter search vector {{dimensionTip}}',
   secondTip: '1. Choose collection and field',
   thirdTip: 'Search Parameters {{metricType}}',
-  vectorPlaceholder: 'Please input your vector value here, e.g. [1, 2, 3, 4]',
   collection: 'loaded collection',
   noCollection: 'No loaded collection',
   field: 'Vector field',
@@ -29,6 +28,7 @@ const searchTrans = {
   consistency: 'Consistency',
   graphNodeHoverTip: 'Double click to explore more',
   inputVectorPlaceHolder: 'Vector or entity id',
+  textPlaceHolder: 'Please input your text here',
   partitionFilter: 'Partition Filter',
   loading: 'Loading...',
 };

client/src/pages/databases/Databases.tsx

@@ -82,7 +82,7 @@ const useStyles = makeStyles((theme: Theme) => ({
 // Databases page(tree and tabs)
 const Databases = () => {
   // context
-  const { database, collections, loading, fetchCollection, ui, setUIPref } =
+  const { database, collections, loading, ui, setUIPref } =
     useContext(dataContext);
 
   // UI state

client/src/pages/databases/collections/StatusAction.tsx

@@ -45,6 +45,7 @@ const useStyles = makeStyles((theme: Theme) => ({
   },
   extraBtn: {
     height: 24,
+    padding: '0 8px',
   },
 }));
 
@@ -55,6 +56,7 @@ const StatusAction: FC<StatusActionType> = props => {
     collection,
     action = () => {},
     showExtraAction,
+    showLoadButton,
     createIndexElement,
   } = props;
 
@@ -134,6 +136,33 @@ const StatusAction: FC<StatusActionType> = props => {
     });
   };
 
+  if (
+    collection.schema &&
+    status === LOADING_STATE.UNLOADED &&
+    collection.schema.hasVectorIndex &&
+    showLoadButton
+  ) {
+    return (
+      <CustomButton
+        startIcon={<Icons.load />}
+        className={classes.extraBtn}
+        variant="contained"
+        tooltip={collectionTrans('clickToLoad')}
+        onClick={() => {
+          setDialog({
+            open: true,
+            type: 'custom',
+            params: {
+              component: <LoadCollectionDialog collection={collection} />,
+            },
+          });
+        }}
+      >
+        {collectionTrans('loadTitle')}
+      </CustomButton>
+    );
+  }
+
   return (
     <div className={classes.root}>
       <CustomToolTip title={noIndex ? noIndexTooltip : tooltip} placement="top">
@@ -166,6 +195,7 @@ const StatusAction: FC<StatusActionType> = props => {
               {btnTrans('vectorSearch')}
             </CustomButton>
           )}
+
           {!collection.schema.hasVectorIndex && createIndexElement}
         </>
       )}

client/src/pages/databases/collections/Types.ts

@@ -1,16 +1,26 @@
 import { Dispatch, SetStateAction } from 'react';
-import { DataTypeEnum } from '@/consts';
+import { DataTypeEnum, FunctionType } from '@/consts';
 
 export interface CollectionCreateProps {
   onCreate?: () => void;
 }
 
+export type FunctionConfig = {
+  name: string;
+  description: string;
+  type: FunctionType;
+  input_field_names: string[];
+  output_field_names: string[];
+  params: Record<string, unknown>;
+};
+
 export interface CollectionCreateParam {
   collection_name: string;
   description: string;
   autoID: boolean;
   fields: CreateField[];
   consistency_level: string;
+  functions: FunctionConfig[];
 }
 
 export type AnalyzerType = 'standard' | 'english' | 'chinese';

client/src/pages/databases/collections/data/CollectionData.tsx

@@ -152,7 +152,7 @@ const CollectionData = (props: CollectionDataProps) => {
   } = useQuery({
     collection,
     consistencyLevel,
-    fields,
+    fields: fields.filter(f => !f.is_function_output),
     onQueryStart: (expr: string = '') => {
       setTableLoading(true);
       if (expr === '') {
@@ -422,15 +422,17 @@ const CollectionData = (props: CollectionDataProps) => {
             <div className="right">
               <CustomMultiSelector
                 className={classes.outputs}
-                options={fields.map(f => {
-                  return {
-                    label:
-                      f.name === DYNAMIC_FIELD
-                        ? searchTrans('dynamicFields')
-                        : f.name,
-                    value: f.name,
-                  };
-                })}
+                options={fields
+                  .filter(f => !f.is_function_output)
+                  .map(f => {
+                    return {
+                      label:
+                        f.name === DYNAMIC_FIELD
+                          ? searchTrans('dynamicFields')
+                          : f.name,
+                      value: f.name,
+                    };
+                  })}
                 values={outputFields}
                 renderValue={selected => (
                   <span>{`${(selected as string[]).length} ${

client/src/pages/databases/collections/schema/CreateIndexDialog.tsx

@@ -1,8 +1,6 @@
 import { useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
-import { CodeLanguageEnum, CodeViewData } from '@/components/code/Types';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
-import CustomSwitch from '@/components/customSwitch/CustomSwitch';
 import { Option } from '@/components/customSelector/Types';
 import {
   INDEX_CONFIG,
@@ -11,38 +9,26 @@ import {
   INDEX_TYPES_ENUM,
   DataTypeEnum,
   DataTypeStringEnum,
-  VectorTypes,
+  VectorTypesString,
 } from '@/consts';
 import { useFormValidation } from '@/hooks';
-import { getCreateIndexJSCode } from '@/utils/code/Js';
-import { getCreateIndexPYCode } from '@/utils/code/Py';
-import { getCreateIndexJavaCode } from '@/utils/code/Java';
-import { formatForm, getMetricOptions, getScalarIndexOption } from '@/utils';
+import {
+  formatForm,
+  getMetricOptions,
+  getScalarIndexOption,
+  isVectorType,
+} from '@/utils';
 import CreateForm from './CreateForm';
 import { IndexType, IndexExtraParam } from './Types';
+import { FieldObject } from '@server/types';
 
 const CreateIndex = (props: {
   collectionName: string;
-  fieldType: DataTypeStringEnum;
-  elementType?: DataTypeStringEnum;
-  dataType: DataTypeEnum;
+  field: FieldObject;
   handleCreate: (params: IndexExtraParam, index_name: string) => void;
   handleCancel: () => void;
-
-  // used for code mode
-  fieldName: string;
-  // used for sizing info
-  dimension: number;
 }) => {
-  const {
-    collectionName,
-    fieldType,
-    elementType,
-    handleCreate,
-    handleCancel,
-    fieldName,
-    dataType,
-  } = props;
+  const { collectionName, handleCreate, handleCancel, field } = props;
 
   const { t: indexTrans } = useTranslation('index');
   const { t: dialogTrans } = useTranslation('dialog');
@@ -53,7 +39,7 @@ const CreateIndex = (props: {
   const defaultIndexType = INDEX_TYPES_ENUM.AUTOINDEX;
 
   const defaultMetricType = useMemo(() => {
-    switch (fieldType) {
+    switch (field.data_type) {
       case DataTypeStringEnum.BinaryVector:
         return METRIC_TYPES_VALUES.HAMMING;
       case DataTypeStringEnum.FloatVector:
@@ -61,11 +47,13 @@ const CreateIndex = (props: {
       case DataTypeStringEnum.BFloat16Vector:
         return METRIC_TYPES_VALUES.COSINE;
       case DataTypeStringEnum.SparseFloatVector:
-        return METRIC_TYPES_VALUES.IP;
+        return field.is_function_output
+          ? METRIC_TYPES_VALUES.BM25
+          : METRIC_TYPES_VALUES.IP;
       default:
         return '';
     }
-  }, [fieldType]);
+  }, [field.data_type]);
 
   const [indexSetting, setIndexSetting] = useState<{
     index_type: IndexType;
@@ -92,9 +80,6 @@ const CreateIndex = (props: {
     cache_dataset_on_device: 'false',
   });
 
-  // control whether show code mode
-  const [showCode, setShowCode] = useState<boolean>(false);
-
   const indexCreateParams = useMemo(() => {
     if (!INDEX_CONFIG[indexSetting.index_type]) {
       return [];
@@ -103,10 +88,10 @@ const CreateIndex = (props: {
   }, [indexSetting.index_type]);
 
   const metricOptions = useMemo(() => {
-    return VectorTypes.includes(dataType)
-      ? getMetricOptions(indexSetting.index_type, dataType)
+    return isVectorType(field)
+      ? getMetricOptions(indexSetting.index_type, field)
       : [];
-  }, [indexSetting.index_type, fieldType]);
+  }, [indexSetting.index_type, field]);
 
   const extraParams = useMemo(() => {
     const params: { [x: string]: string } = {};
@@ -133,8 +118,8 @@ const CreateIndex = (props: {
     const autoOption = getOptions('AUTOINDEX', INDEX_OPTIONS_MAP['AUTOINDEX']);
     let options = [];
 
-    if (VectorTypes.includes(dataType)) {
-      switch (fieldType) {
+    if (isVectorType(field)) {
+      switch (field.data_type) {
         case DataTypeStringEnum.BinaryVector:
           options = [
             ...getOptions(
@@ -166,18 +151,15 @@ const CreateIndex = (props: {
       }
     } else {
       options = [
-        ...getOptions(
-          indexTrans('scalar'),
-          getScalarIndexOption(fieldType, elementType)
-        ),
+        ...getOptions(indexTrans('scalar'), getScalarIndexOption(field)),
       ];
     }
 
     return [...autoOption, ...options];
-  }, [fieldType, dataType, fieldName]);
+  }, [field]);
 
   const checkedForm = useMemo(() => {
-    if (!VectorTypes.includes(dataType)) {
+    if (!isVectorType(field)) {
       return [];
     }
     const paramsForm: any = { metric_type: indexSetting.metric_type };
@@ -186,47 +168,7 @@ const CreateIndex = (props: {
     });
     const form = formatForm(paramsForm);
     return form;
-  }, [indexSetting, indexCreateParams, fieldType]);
-
-  /**
-   * create index code mode
-   */
-  const codeBlockData: CodeViewData[] = useMemo(() => {
-    const isScalarField = !VectorTypes.includes(dataType);
-    const getCodeParams = {
-      collectionName,
-      fieldName,
-      extraParams,
-      isScalarField,
-      indexName: indexSetting.index_name,
-      metricType: indexSetting.metric_type,
-      indexType: indexSetting.index_type,
-    };
-    return [
-      {
-        label: commonTrans('py'),
-        language: CodeLanguageEnum.python,
-        code: getCreateIndexPYCode(getCodeParams),
-      },
-      {
-        label: commonTrans('java'),
-        language: CodeLanguageEnum.java,
-        code: getCreateIndexJavaCode(getCodeParams),
-      },
-      {
-        label: commonTrans('js'),
-        language: CodeLanguageEnum.javascript,
-        code: getCreateIndexJSCode(getCodeParams),
-      },
-    ];
-  }, [
-    commonTrans,
-    extraParams,
-    collectionName,
-    fieldName,
-    indexSetting.index_name,
-    fieldType,
-  ]);
+  }, [indexSetting, indexCreateParams, field]);
 
   const {
     validation,
@@ -262,24 +204,16 @@ const CreateIndex = (props: {
     await handleCreate(extraParams, indexSetting.index_name);
   };
 
-  const handleShowCode = (event: React.ChangeEvent<{ checked: boolean }>) => {
-    const isChecked = event.target.checked;
-    setShowCode(isChecked);
-  };
-
   return (
     <DialogTemplate
       title={dialogTrans('createTitle', {
         type: indexTrans('index'),
-        name: fieldName,
+        name: field.name,
       })}
       handleClose={handleCancel}
       confirmLabel={btnTrans('create')}
       handleConfirm={handleCreateIndex}
       confirmDisabled={disabled}
-      leftActions={<CustomSwitch onChange={handleShowCode} />}
-      showCode={showCode}
-      codeBlocksData={codeBlockData}
     >
       <>
         <CreateForm

client/src/pages/databases/collections/schema/IndexTypeElement.tsx

@@ -8,15 +8,12 @@ import Icons from '@/components/icons/Icons';
 import DeleteTemplate from '@/components/customDialog/DeleteDialogTemplate';
 import StatusIcon, { LoadingType } from '@/components/status/StatusIcon';
 import { IndexState } from '@/types/Milvus';
-import {
-  NONE_INDEXABLE_DATA_TYPES,
-  DataTypeStringEnum,
-  DataTypeEnum,
-} from '@/consts';
+import { NONE_INDEXABLE_DATA_TYPES, DataTypeStringEnum } from '@/consts';
 import CreateIndexDialog from './CreateIndexDialog';
 import { FieldObject } from '@server/types';
 import CustomButton from '@/components/customButton/CustomButton';
 import { makeStyles } from '@mui/styles';
+import { isVectorType } from '@/utils';
 
 const useStyles = makeStyles((theme: Theme) => ({
   wrapper: {
@@ -31,10 +28,18 @@ const useStyles = makeStyles((theme: Theme) => ({
     alignItems: 'center',
     whiteSpace: 'nowrap',
     color: theme.palette.primary.main,
-    height: 24,
+    height: 26,
+    fontSize: 13,
+    border: `1px solid transparent`,
     '&:hover': {
       cursor: 'pointer',
     },
+    '&.outline': {
+      border: `1px dashed ${theme.palette.primary.main}`,
+    },
+    '& svg': {
+      width: 15,
+    },
   },
   btnDisabled: {
     color: theme.palette.text.secondary,
@@ -67,7 +72,7 @@ const IndexTypeElement: FC<{
   disabled?: boolean;
   disabledTooltip?: string;
   cb?: (collectionName: string) => void;
-}> = ({ field, collectionName, cb, disabled, disabledTooltip }) => {
+}> = ({ field, collectionName, cb, disabled }) => {
   const { createIndex, dropIndex } = useContext(dataContext);
 
   const classes = useStyles();
@@ -75,7 +80,6 @@ const IndexTypeElement: FC<{
   const { t: indexTrans } = useTranslation('index');
   const { t: dialogTrans } = useTranslation('dialog');
   const { t: successTrans } = useTranslation('success');
-  const { t: collectionTrans } = useTranslation('collection');
   const { t: btnTrans } = useTranslation('btn');
 
   const { setDialog, handleCloseDialog, openSnackBar } =
@@ -108,11 +112,7 @@ const IndexTypeElement: FC<{
         component: (
           <CreateIndexDialog
             collectionName={collectionName}
-            fieldName={field.name}
-            dataType={field.dataType as unknown as DataTypeEnum}
-            fieldType={field.data_type as DataTypeStringEnum}
-            elementType={field.element_type as DataTypeStringEnum}
-            dimension={Number(field.dimension)}
+            field={field}
             handleCancel={handleCloseDialog}
             handleCreate={requestCreateIndex}
           />
@@ -192,14 +192,14 @@ const IndexTypeElement: FC<{
     }
 
     if (!field.index) {
+      const isVector = isVectorType(field);
       return (
         <CustomButton
-          startIcon={<Icons.add />}
-          className={classes.btn}
-          tooltip={collectionTrans('clickToCreateVectorIndex')}
+          startIcon={<Icons.addOutline />}
+          className={`${classes.btn}${isVector ? ' outline' : ''}`}
           onClick={e => handleCreate(e)}
         >
-          {btnTrans('createIndex')}
+          {btnTrans(isVector ? 'createVectorIndex' : 'createScalarIndex')}
         </CustomButton>
       );
     }

client/src/pages/databases/collections/schema/Schema.tsx

@@ -61,7 +61,7 @@ const Overview = () => {
                 className={classes.primaryKeyChip}
                 title={collectionTrans('idFieldName')}
               >
-                <Icons.key classes={{ root: 'key' }} />
+                <Chip className={classes.chip} size="small" label="ID" />
               </div>
             ) : null}
             {f.is_partition_key ? (
@@ -71,13 +71,6 @@ const Overview = () => {
                 label="Partition key"
               />
             ) : null}
-            {f.autoID ? (
-              <Chip
-                className={classes.chip}
-                size="small"
-                label={collectionTrans('autoId')}
-              />
-            ) : null}
             {findKeyValue(f.type_params, 'enable_match') ? (
               <Chip
                 className={classes.chip}
@@ -111,6 +104,32 @@ const Overview = () => {
                 />
               </Tooltip>
             ) : null}
+
+            {f.function ? (
+              <Tooltip title={JSON.stringify(f.function)} arrow>
+                <Chip
+                  className={classes.chip}
+                  size="small"
+                  label={`
+                    ${
+                      f.is_function_output
+                        ? `<- ${f.function.type}(${f.function.input_field_names})`
+                        : ` ${collectionTrans('function')}: ${f.function.type}`
+                    }`}
+                  onClick={() => {
+                    const textToCopy = JSON.stringify(f.function);
+                    navigator.clipboard
+                      .writeText(textToCopy as string)
+                      .then(() => {
+                        alert('Copied to clipboard!');
+                      })
+                      .catch(err => {
+                        alert('Failed to copy: ' + err);
+                      });
+                  }}
+                />
+              </Tooltip>
+            ) : null}
           </div>
         );
       },
@@ -239,6 +258,11 @@ const Overview = () => {
   const enableModifyReplica =
     data && data.queryNodes && data.queryNodes.length > 1;
 
+  // if is autoID enabled
+  const isAutoIDEnabled = collection?.schema?.fields.some(
+    f => f.autoID === true
+  );
+
   // get loading state label
   return (
     <section className={classes.wrapper}>
@@ -304,7 +328,8 @@ const Overview = () => {
                 status={collection.status}
                 percentage={collection.loadedPercentage}
                 collection={collection}
-                showExtraAction={true}
+                showExtraAction={false}
+                showLoadButton={true}
                 createIndexElement={CreateIndexElement}
               />
               <Typography variant="h5">
@@ -362,6 +387,13 @@ const Overview = () => {
                 {collectionTrans('features')}
               </Typography>
               <Typography variant="h6">
+                {isAutoIDEnabled ? (
+                  <Chip
+                    className={`${classes.chip} ${classes.featureChip}`}
+                    label={collectionTrans('autoId')}
+                    size="small"
+                  />
+                ) : null}
                 <Tooltip
                   title={
                     consistencyTooltipsMap[collection.consistency_level!] || ''

client/src/pages/databases/collections/schema/Styles.tsx

@@ -73,19 +73,18 @@ export const useStyles = makeStyles((theme: Theme) => ({
   },
   primaryKeyChip: {
     fontSize: '8px',
-    position: 'relative',
-    top: '3px',
-    color: 'grey',
   },
   chip: {
     fontSize: '12px',
     color: theme.palette.text.primary,
     border: 'none',
     cursor: 'normal',
+    marginRight: 4,
+    marginLeft: 4,
   },
   featureChip: {
-    marginRight: 4,
     border: 'none',
+    marginLeft: 0,
   },
   nameWrapper: {
     display: 'flex',

client/src/pages/databases/collections/search/Search.tsx

@@ -18,7 +18,7 @@ import { getLabelDisplayedRows } from '@/pages/search/Utils';
 import { useSearchResult, usePaginationHook } from '@/hooks';
 import { getQueryStyles } from './Styles';
 import SearchGlobalParams from './SearchGlobalParams';
-import VectorInputBox from './VectorInputBox';
+import VectorInputBox from './SearchInputBox';
 import StatusIcon, { LoadingType } from '@/components/status/StatusIcon';
 import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
 import CustomInput from '@/components/customInput/CustomInput';
@@ -138,7 +138,7 @@ const Search = (props: CollectionDataProps) => {
   }, [JSON.stringify(searchParams)]);
 
   // on vector input change, update the search params
-  const onVectorInputChange = useCallback(
+  const onSearchInputChange = useCallback(
     (anns_field: string, value: string) => {
       const s = cloneObj(searchParams) as SearchParamsType;
       const target = s.searchParams.find((sp: SearchSingleParams) => {
@@ -442,7 +442,11 @@ const Search = (props: CollectionDataProps) => {
                             s.data.length > 0 ? 'bold' : ''
                           }`}
                         >
-                          {field.name}
+                          {field.is_function_output
+                            ? `${field.name}<=${
+                                field.function!.input_field_names[0]
+                              }`
+                            : field.name}
                         </Typography>
                         <Typography className="vector-type">
                           {formatFieldType(field)}
@@ -454,8 +458,9 @@ const Search = (props: CollectionDataProps) => {
                   <AccordionDetails className={classes.accordionDetail}>
                     <VectorInputBox
                       searchParams={s}
-                      onChange={onVectorInputChange}
+                      onChange={onSearchInputChange}
                       collection={collection}
+                      type={field.is_function_output ? 'text' : 'vector'}
                     />
 
                     <Typography className="text">

client/src/pages/databases/collections/search/VectorInputBox.tsx → client/src/pages/databases/collections/search/SearchInputBox.tsx

@@ -7,118 +7,31 @@ import { indentUnit } from '@codemirror/language';
 import { minimalSetup } from 'codemirror';
 import { javascript } from '@codemirror/lang-javascript';
 import { linter, Diagnostic } from '@codemirror/lint';
-import { CollectionFullObject, FieldObject } from '@server/types';
+import { CollectionFullObject } from '@server/types';
 import { CollectionService } from '@/http';
 import { DataTypeStringEnum } from '@/consts';
 import { SearchSingleParams } from '../../types';
-import { isSparseVector, transformObjStrToJSONStr } from '@/utils';
 import { getQueryStyles } from './Styles';
 import { useTheme } from '@mui/material';
 import { githubLight } from '@ddietr/codemirror-themes/github-light';
 import { githubDark } from '@ddietr/codemirror-themes/github-dark';
+import { Validator } from './utils';
 
-const floatVectorValidator = (text: string, field: FieldObject) => {
-  try {
-    const value = JSON.parse(text);
-    const dim = field.dimension;
-    if (!Array.isArray(value)) {
-      return {
-        valid: false,
-        message: `Not an array`,
-      };
-    }
-
-    if (Array.isArray(value) && value.length !== dim) {
-      return {
-        valid: false,
-        value: undefined,
-        message: `Dimension ${value.length} is not equal to ${dim} `,
-      };
-    }
-
-    return { valid: true, message: ``, value: value };
-  } catch (e: any) {
-    return {
-      valid: false,
-      message: `Wrong Float Vector format, it should be an array of ${field.dimension} numbers`,
-    };
-  }
-};
-
-const binaryVectorValidator = (text: string, field: FieldObject) => {
-  try {
-    const value = JSON.parse(text);
-    const dim = field.dimension;
-    if (!Array.isArray(value)) {
-      return {
-        valid: false,
-        message: `Not an array`,
-      };
-    }
-
-    if (Array.isArray(value) && value.length !== dim / 8) {
-      return {
-        valid: false,
-        value: undefined,
-        message: `Dimension ${value.length} is not equal to ${dim / 8} `,
-      };
-    }
-
-    return { valid: true, message: ``, value: value };
-  } catch (e: any) {
-    return {
-      valid: false,
-      message: `Wrong Binary Vector format, it should be an array of ${
-        field.dimension / 8
-      } numbers`,
-    };
-  }
-};
-
-const sparseVectorValidator = (text: string, field: FieldObject) => {
-  if (!isSparseVector(text)) {
-    return {
-      valid: false,
-      value: undefined,
-      message: `Incorrect Sparse Vector format, it should be like {1: 0.1, 3: 0.2}`,
-    };
-  }
-  try {
-    JSON.parse(transformObjStrToJSONStr(text));
-    return {
-      valid: true,
-      message: ``,
-    };
-  } catch (e: any) {
-    return {
-      valid: false,
-      message: `Wrong Sparse Vector format`,
-    };
-  }
-};
-
-const Validator = {
-  [DataTypeStringEnum.FloatVector]: floatVectorValidator,
-  [DataTypeStringEnum.BinaryVector]: binaryVectorValidator,
-  [DataTypeStringEnum.Float16Vector]: floatVectorValidator,
-  [DataTypeStringEnum.BFloat16Vector]: floatVectorValidator,
-  [DataTypeStringEnum.SparseFloatVector]: sparseVectorValidator,
-};
-
-export type VectorInputBoxProps = {
+export type SearchInputBoxProps = {
   onChange: (anns_field: string, value: string) => void;
   searchParams: SearchSingleParams;
   collection: CollectionFullObject;
+  type?: 'vector' | 'text';
 };
 
 let queryTimeout: NodeJS.Timeout;
 
-export default function VectorInputBox(props: VectorInputBoxProps) {
+export default function SearchInputBox(props: SearchInputBoxProps) {
   const theme = useTheme();
   const { t: searchTrans } = useTranslation('search');
 
   // props
-  const { searchParams, onChange, collection } = props;
+  const { searchParams, onChange, collection, type } = props;
   const { field, data } = searchParams;
 
   // classes
@@ -187,12 +100,64 @@ export default function VectorInputBox(props: VectorInputBoxProps) {
   // create editor
   useEffect(() => {
     if (!editor.current) {
-      const startState = EditorState.create({
-        doc: data,
-        extensions: [
-          minimalSetup,
+      // update outside data timeout handler
+      let updateTimeout: NodeJS.Timeout;
+
+      let extensions = [
+        minimalSetup,
+        placeholder(
+          searchTrans(
+            type === 'text' ? 'textPlaceHolder' : 'inputVectorPlaceHolder'
+          )
+        ),
+        keymap.of([{ key: 'Tab', run: insertTab }]), // fix tab behaviour
+        indentUnit.of('    '), // fix tab indentation
+        EditorView.theme({
+          '&.cm-editor': {
+            '&.cm-focused': {
+              outline: 'none',
+            },
+          },
+          '.cm-content': {
+            fontSize: '12px',
+            minHeight: '124px',
+          },
+          '.cm-gutters': {
+            display: 'none',
+          },
+        }),
+        EditorView.lineWrapping,
+        EditorView.updateListener.of(update => {
+          if (update.docChanged) {
+            if (queryTimeout || updateTimeout) {
+              clearTimeout(queryTimeout);
+              clearTimeout(updateTimeout);
+            }
+
+            updateTimeout = setTimeout(() => {
+              // get text
+              const text = update.state.doc.toString();
+              // validate text
+              const { valid } = validator(text, fieldRef.current);
+              // if valid, update search params
+              if (valid || text === '' || type === 'text') {
+                onChangeRef.current(searchParams.anns_field, text);
+              } else {
+                getVectorById(text);
+              }
+            }, 500);
+          }
+          if (update.focusChanged) {
+            editorEl.current?.classList.toggle('focused', update.view.hasFocus);
+          }
+        }),
+      ];
+
+      if (type === 'vector') {
+        extensions = [
+          ...extensions,
           javascript(),
-          placeholder(searchTrans('inputVectorPlaceHolder')),
+
           linter(view => {
             const text = view.state.doc.toString();
 
@@ -227,67 +192,37 @@ export default function VectorInputBox(props: VectorInputBoxProps) {
               return [];
             }
           }),
-          keymap.of([{ key: 'Tab', run: insertTab }]), // fix tab behaviour
-          indentUnit.of('    '), // fix tab indentation
-          EditorView.theme({
-            '&.cm-editor': {
-              '&.cm-focused': {
-                outline: 'none',
-              },
-            },
-            '.cm-content': {
-              fontSize: '12px',
-              minHeight: '124px',
-            },
-            '.cm-gutters': {
-              display: 'none',
-            },
-          }),
-          EditorView.lineWrapping,
-          EditorView.updateListener.of(update => {
-            if (update.docChanged) {
-              if (queryTimeout) {
-                clearTimeout(queryTimeout);
-              }
-              const text = update.state.doc.toString();
+        ];
+      }
 
-              const { valid } = validator(text, fieldRef.current);
-              if (valid || text === '') {
-                onChangeRef.current(searchParams.anns_field, text);
-              } else {
-                getVectorById(text);
-              }
-            }
-            if (update.focusChanged) {
-              editorEl.current?.classList.toggle(
-                'focused',
-                update.view.hasFocus
-              );
-            }
-          }),
-        ],
+      // create editor
+      const startState = EditorState.create({
+        doc: data,
+        extensions,
       });
 
+      // create editor view
       const view = new EditorView({
         state: startState,
         parent: editorEl.current!,
       });
 
+      // set editor ref
       editor.current = view;
-
-      // focus editor, the cursor will be at the end of the text
-      const endPos = editor.current.state.doc.length;
-      editor.current.dispatch({
-        selection: { anchor: endPos },
-      });
-
-      editor.current.focus();
-
-      return () => {
-        view.destroy();
-        editor.current = undefined;
-      };
+    } else {
+      if (editor.current.state.doc.toString() !== data) {
+        console.log('not equal');
+        editor.current.dispatch({
+          changes: {
+            from: 0,
+            to: editor.current.state.doc.length,
+            insert: data,
+          },
+        });
+      }
     }
+
+    return () => {};
   }, [JSON.stringify({ field, data })]);
 
   useEffect(() => {
@@ -303,5 +238,5 @@ export default function VectorInputBox(props: VectorInputBoxProps) {
     }
   }, [theme.palette.mode]);
 
-  return <div className={classes.vectorInputBox} ref={editorEl}></div>;
+  return <div className={classes.searchInputBox} ref={editorEl}></div>;
 }

client/src/pages/databases/collections/search/Styles.ts

@@ -18,7 +18,7 @@ export const getQueryStyles = makeStyles((theme: Theme) => ({
 
   accordions: {
     display: 'flex',
-    width: '220px',
+    width: '230px',
     flexDirection: 'column',
     flexShrink: 0,
     padding: '0 8px 8px 0',
@@ -107,12 +107,12 @@ export const getQueryStyles = makeStyles((theme: Theme) => ({
         marginLeft: '4px',
         fontSize: '10px',
         fontWeight: 600,
-        color: theme.palette.primary.light,
+        color: theme.palette.secondary.main,
       },
     },
   },
 
-  vectorInputBox: {
+  searchInputBox: {
     height: '124px',
     margin: '0 0 8px 0',
     overflow: 'auto',

client/src/pages/databases/collections/search/utils.ts

@@ -0,0 +1,91 @@
+import { isSparseVector, transformObjStrToJSONStr } from '@/utils';
+import { FieldObject } from '@server/types';
+import { DataTypeStringEnum } from '@/consts';
+
+const floatVectorValidator = (text: string, field: FieldObject) => {
+  try {
+    const value = JSON.parse(text);
+    const dim = field.dimension;
+    if (!Array.isArray(value)) {
+      return {
+        valid: false,
+        message: `Not an array`,
+      };
+    }
+
+    if (Array.isArray(value) && value.length !== dim) {
+      return {
+        valid: false,
+        value: undefined,
+        message: `Dimension ${value.length} is not equal to ${dim} `,
+      };
+    }
+
+    return { valid: true, message: ``, value: value };
+  } catch (e: any) {
+    return {
+      valid: false,
+      message: `Wrong Float Vector format, it should be an array of ${field.dimension} numbers`,
+    };
+  }
+};
+
+const binaryVectorValidator = (text: string, field: FieldObject) => {
+  try {
+    const value = JSON.parse(text);
+    const dim = field.dimension;
+    if (!Array.isArray(value)) {
+      return {
+        valid: false,
+        message: `Not an array`,
+      };
+    }
+
+    if (Array.isArray(value) && value.length !== dim / 8) {
+      return {
+        valid: false,
+        value: undefined,
+        message: `Dimension ${value.length} is not equal to ${dim / 8} `,
+      };
+    }
+
+    return { valid: true, message: ``, value: value };
+  } catch (e: any) {
+    return {
+      valid: false,
+      message: `Wrong Binary Vector format, it should be an array of ${
+        field.dimension / 8
+      } numbers`,
+    };
+  }
+};
+
+const sparseVectorValidator = (text: string, field: FieldObject) => {
+  if (!isSparseVector(text)) {
+    return {
+      valid: false,
+      value: undefined,
+      message: `Incorrect Sparse Vector format, it should be like {1: 0.1, 3: 0.2}`,
+    };
+  }
+  try {
+    JSON.parse(transformObjStrToJSONStr(text));
+    return {
+      valid: true,
+      message: ``,
+    };
+  } catch (e: any) {
+    return {
+      valid: false,
+      message: `Wrong Sparse Vector format`,
+    };
+  }
+};
+
+export const Validator = {
+  [DataTypeStringEnum.FloatVector]: floatVectorValidator,
+  [DataTypeStringEnum.BinaryVector]: binaryVectorValidator,
+  [DataTypeStringEnum.Float16Vector]: floatVectorValidator,
+  [DataTypeStringEnum.BFloat16Vector]: floatVectorValidator,
+  [DataTypeStringEnum.SparseFloatVector]: sparseVectorValidator,
+};

client/src/pages/dialogs/CreateCollectionDialog.tsx

@@ -8,14 +8,19 @@ import { ITextfieldConfig } from '@/components/customInput/Types';
 import { rootContext, dataContext } from '@/context';
 import { useFormValidation } from '@/hooks';
 import { formatForm, getAnalyzerParams, TypeEnum } from '@/utils';
-import { DataTypeEnum, ConsistencyLevelEnum, DEFAULT_ATTU_DIM } from '@/consts';
-import CreateFields from '../databases/collections/CreateFields';
+import {
+  DataTypeEnum,
+  ConsistencyLevelEnum,
+  DEFAULT_ATTU_DIM,
+  FunctionType,
+} from '@/consts';
+import CreateFields from './create/CreateFields';
 import {
   CollectionCreateParam,
   CollectionCreateProps,
   CreateField,
 } from '../databases/collections/Types';
-import { CONSISTENCY_LEVEL_OPTIONS } from '../databases/collections/Constants';
+import { CONSISTENCY_LEVEL_OPTIONS } from './create/Constants';
 import { makeStyles } from '@mui/styles';
 
 const useStyles = makeStyles((theme: Theme) => ({
@@ -29,7 +34,6 @@ const useStyles = makeStyles((theme: Theme) => ({
       marginBottom: '0',
     },
     '& legend': {
-      marginBottom: theme.spacing(1),
       lineHeight: '20px',
       fontSize: '14px',
     },
@@ -49,7 +53,7 @@ const useStyles = makeStyles((theme: Theme) => ({
     marginTop: theme.spacing(2),
   },
   dialog: {
-    minWidth: 820,
+    minWidth: 880,
   },
 }));
 
@@ -191,6 +195,8 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
   ];
 
   const handleCreateCollection = async () => {
+    // function output fields
+    const fnOutputFields: CreateField[] = [];
     const param: CollectionCreateParam = {
       ...form,
       fields: fields.map(v => {
@@ -203,6 +209,9 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
           data_type: v.data_type,
         };
 
+        // remove unused id
+        delete data.id;
+
         // if we need
         if (typeof v.dim !== undefined && !isNaN(Number(v.dim))) {
           data.dim = Number(v.dim);
@@ -218,23 +227,71 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
           data.max_capacity = Number(v.max_capacity);
         }
 
-        if (v.analyzer_params && v.enable_analyzer) {
+        // handle BM25 row
+        if (data.data_type === DataTypeEnum.VarCharBM25) {
+          data.data_type = DataTypeEnum.VarChar;
+          data.enable_analyzer = true;
+          data.analyzer_params = data.analyzer_params || 'standard';
+          // create sparse field
+          const sparseField = {
+            name: `${data.name}_embeddings`,
+            is_primary_key: false,
+            data_type: DataTypeEnum.SparseFloatVector,
+            description: `fn BM25(${data.name}) -> embeddings`,
+            is_function_output: true,
+          };
+          // push sparse field to fields
+          fnOutputFields.push(sparseField);
+        }
+
+        if (data.analyzer_params && data.enable_analyzer) {
           // if analyzer_params is string, we need to use default value
-          data.analyzer_params = getAnalyzerParams(v.analyzer_params);
+          data.analyzer_params = getAnalyzerParams(data.analyzer_params);
+        } else {
+          delete data.analyzer_params;
+          delete data.enable_analyzer;
         }
 
-        v.is_primary_key && (data.autoID = form.autoID);
+        data.is_primary_key && (data.autoID = form.autoID);
 
         // delete sparse vector dime
         if (data.data_type === DataTypeEnum.SparseFloatVector) {
           delete data.dim;
         }
+        // delete analyzer if not varchar
+        if (
+          data.data_type !== DataTypeEnum.VarChar &&
+          data.data_type === DataTypeEnum.Array &&
+          data.element_type !== DataTypeEnum.VarChar
+        ) {
+          delete data.enable_analyzer;
+          delete data.analyzer_params;
+          delete data.max_length;
+        }
 
         return data;
       }),
+      functions: [],
       consistency_level: consistencyLevel,
     };
 
+    // push sparse fields to param.fields
+    param.fields.push(...fnOutputFields);
+
+    // build functions
+    fnOutputFields.forEach((field, index) => {
+      const [input] = (field.name as string).split('_');
+      const functionParam = {
+        name: `BM25_${index}`,
+        description: `${input} BM25 function`,
+        type: FunctionType.BM25,
+        input_field_names: [input],
+        output_field_names: [field.name as string],
+        params: {},
+      };
+      param.functions.push(functionParam);
+    });
+
     // create collection
     await createCollection({
       ...param,
@@ -276,7 +333,7 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
         </fieldset>
 
         <fieldset className={classes.fieldset}>
-          <legend>{collectionTrans('schema')}</legend>
+          {/* <legend>{collectionTrans('schema')}</legend> */}
           <CreateFields
             fields={fields}
             setFields={setFields}

client/src/pages/databases/collections/Constants.ts → client/src/pages/dialogs/create/Constants.ts

@@ -41,10 +41,13 @@ export const VECTOR_FIELDS_OPTIONS: LabelValuePair[] = [
     label: 'Sparse Vector',
     value: DataTypeEnum.SparseFloatVector,
   },
+  {
+    label: 'VarChar(BM25)',
+    value: DataTypeEnum.VarCharBM25,
+  },
 ];
 
 export const ALL_OPTIONS: LabelValuePair[] = [
-  ...VECTOR_FIELDS_OPTIONS,
   {
     label: 'Int8',
     value: DataTypeEnum.Int8,

client/src/pages/databases/collections/CreateFields.tsx → client/src/pages/dialogs/create/CreateFields.tsx

@@ -26,7 +26,11 @@ import {
   VECTOR_FIELDS_OPTIONS,
   ANALYZER_OPTIONS,
 } from './Constants';
-import { CreateFieldsProps, CreateFieldType, FieldType } from './Types';
+import {
+  CreateFieldsProps,
+  CreateFieldType,
+  FieldType,
+} from '../../databases/collections/Types';
 import { DataTypeEnum, VectorTypes } from '@/consts';
 import {
   DEFAULT_ATTU_DIM,
@@ -39,17 +43,23 @@ import CustomIconButton from '@/components/customButton/CustomIconButton';
 import EditAnalyzerDialog from '@/pages/dialogs/EditAnalyzerDialog';
 
 const useStyles = makeStyles((theme: Theme) => ({
-  optionalWrapper: {
+  scalarFieldsWrapper: {
     width: '100%',
     paddingRight: theme.spacing(1),
     overflowY: 'auto',
   },
+  title: {
+    '& button': {
+      position: 'relative',
+      top: '-1px',
+      marginLeft: 4,
+    },
+  },
   rowWrapper: {
     display: 'flex',
     flexWrap: 'nowrap',
     alignItems: 'center',
     gap: '8px',
-    flex: '1 0 auto',
     marginBottom: 4,
     '& .MuiFormLabel-root': {
       fontSize: 14,
@@ -74,6 +84,10 @@ const useStyles = makeStyles((theme: Theme) => ({
     width: '150px',
     marginTop: '-20px',
   },
+  smallSelect: {
+    width: '105px',
+    marginTop: '-20px',
+  },
   autoIdSelect: {
     width: '120px',
     marginTop: '-20px',
@@ -162,7 +176,7 @@ const CreateFields: FC<CreateFieldsProps> = ({
   const AddIcon = icons.addOutline;
   const RemoveIcon = icons.remove;
 
-  const { requiredFields, optionalFields } = useMemo(
+  const { requiredFields, scalarFields } = useMemo(
     () =>
       fields.reduce(
         (acc, field) => {
@@ -170,10 +184,11 @@ const CreateFields: FC<CreateFieldsProps> = ({
           const requiredTypes: CreateFieldType[] = [
             'primaryKey',
             'defaultVector',
+            'vector',
           ];
           const key = requiredTypes.includes(createType)
             ? 'requiredFields'
-            : 'optionalFields';
+            : 'scalarFields';
 
           acc[key].push({
             ...field,
@@ -184,7 +199,7 @@ const CreateFields: FC<CreateFieldsProps> = ({
         },
         {
           requiredFields: [] as FieldType[],
-          optionalFields: [] as FieldType[],
+          scalarFields: [] as FieldType[],
         }
       ),
 
@@ -192,17 +207,18 @@ const CreateFields: FC<CreateFieldsProps> = ({
   );
 
   const getSelector = (
-    type: 'all' | 'vector' | 'element' | 'primaryKey',
+    type: 'scalar' | 'vector' | 'element' | 'primaryKey',
     label: string,
     value: number,
-    onChange: (value: DataTypeEnum) => void
+    onChange: (value: DataTypeEnum) => void,
+    className: string = classes.select
   ) => {
     let _options = ALL_OPTIONS;
     switch (type) {
       case 'primaryKey':
         _options = PRIMARY_FIELDS_OPTIONS;
         break;
-      case 'all':
+      case 'scalar':
         _options = ALL_OPTIONS;
         break;
       case 'vector':
@@ -213,6 +229,7 @@ const CreateFields: FC<CreateFieldsProps> = ({
           d =>
             d.label !== 'Array' &&
             d.label !== 'JSON' &&
+            d.label !== 'VarChar(BM25)' &&
             !d.label.includes('Vector')
         );
         break;
@@ -222,7 +239,7 @@ const CreateFields: FC<CreateFieldsProps> = ({
 
     return (
       <CustomSelector
-        wrapperClass={classes.select}
+        wrapperClass={className}
         options={_options}
         size="small"
         onChange={e => {
@@ -557,13 +574,17 @@ const CreateFields: FC<CreateFieldsProps> = ({
     return (
       <div className={classes.analyzerInput}>
         <Checkbox
-          checked={!!field.enable_analyzer}
+          checked={
+            !!field.enable_analyzer ||
+            field.data_type === DataTypeEnum.VarCharBM25
+          }
           size="small"
           onChange={() => {
             changeFields(field.id!, {
               enable_analyzer: !field.enable_analyzer,
             });
           }}
+          disabled={field.data_type === DataTypeEnum.VarCharBM25}
         />
         <CustomSelector
           wrapperClass="select"
@@ -572,13 +593,19 @@ const CreateFields: FC<CreateFieldsProps> = ({
           onChange={e => {
             changeFields(field.id!, { analyzer_params: e.target.value });
           }}
-          disabled={!field.enable_analyzer}
+          disabled={
+            !field.enable_analyzer &&
+            field.data_type !== DataTypeEnum.VarCharBM25
+          }
           value={analyzer}
           variant="filled"
           label={collectionTrans('analyzer')}
         />
         <CustomIconButton
-          disabled={!field.enable_analyzer}
+          disabled={
+            !field.enable_analyzer &&
+            field.data_type !== DataTypeEnum.VarCharBM25
+          }
           onClick={() => {
             setDialog2({
               open: true,
@@ -625,6 +652,7 @@ const CreateFields: FC<CreateFieldsProps> = ({
       // remove varchar params, if not varchar
       if (
         updatedField.data_type !== DataTypeEnum.VarChar &&
+        updatedField.data_type !== DataTypeEnum.VarCharBM25 &&
         updatedField.element_type !== DataTypeEnum.VarChar
       ) {
         delete updatedField.max_length;
@@ -647,15 +675,17 @@ const CreateFields: FC<CreateFieldsProps> = ({
     setFields(newFields);
   };
 
-  const handleAddNewField = (index: number) => {
+  const handleAddNewField = (index: number, type = DataTypeEnum.Int16) => {
     const id = generateId();
     const newDefaultItem: FieldType = {
       name: '',
-      data_type: DataTypeEnum.Int16,
+      data_type: type,
       is_primary_key: false,
       description: '',
       isDefault: false,
       dim: DEFAULT_ATTU_DIM,
+      max_length: DEFAULT_ATTU_VARCHAR_MAX_LENGTH,
+      enable_analyzer: type === DataTypeEnum.VarCharBM25,
       id,
     };
     const newValidation = {
@@ -740,7 +770,7 @@ const CreateFields: FC<CreateFieldsProps> = ({
         {generateDimension(field)}
         {generateDesc(field)}
         <IconButton
-          onClick={() => handleAddNewField(index)}
+          onClick={() => handleAddNewField(index, field.data_type)}
           classes={{ root: classes.iconBtn }}
           aria-label="add"
           size="large"
@@ -751,7 +781,7 @@ const CreateFields: FC<CreateFieldsProps> = ({
     );
   };
 
-  const generateNonRequiredRow = (
+  const generateScalarFieldRow = (
     field: FieldType,
     index: number,
     fields: FieldType[]
@@ -776,10 +806,12 @@ const CreateFields: FC<CreateFieldsProps> = ({
       <div className={`${classes.rowWrapper}`}>
         {generateFieldName(field)}
         {getSelector(
-          'all',
+          'scalar',
           collectionTrans('fieldType'),
           field.data_type,
-          (value: DataTypeEnum) => changeFields(field.id!, { data_type: value })
+          (value: DataTypeEnum) =>
+            changeFields(field.id!, { data_type: value }),
+          classes.smallSelect
         )}
 
         {isArray
@@ -788,7 +820,8 @@ const CreateFields: FC<CreateFieldsProps> = ({
               collectionTrans('elementType'),
               field.element_type || DEFAULT_ATTU_ELEMENT_TYPE,
               (value: DataTypeEnum) =>
-                changeFields(field.id!, { element_type: value })
+                changeFields(field.id!, { element_type: value }),
+              classes.smallSelect
             )
           : null}
 
@@ -814,7 +847,7 @@ const CreateFields: FC<CreateFieldsProps> = ({
 
         <IconButton
           onClick={() => {
-            handleAddNewField(index);
+            handleAddNewField(index, field.data_type);
           }}
           classes={{ root: classes.iconBtn }}
           aria-label="add"
@@ -837,23 +870,36 @@ const CreateFields: FC<CreateFieldsProps> = ({
     );
   };
 
-  const generateVectorRow = (field: FieldType, index: number) => {
+  const generateFunctionRow = (
+    field: FieldType,
+    index: number,
+    fields: FieldType[],
+    requiredFields: FieldType[]
+  ) => {
     return (
       <div className={`${classes.rowWrapper}`}>
         {generateFieldName(field)}
         {getSelector(
-          'all',
+          'vector',
           collectionTrans('fieldType'),
           field.data_type,
           (value: DataTypeEnum) => changeFields(field.id!, { data_type: value })
         )}
 
-        {generateDimension(field)}
+        {generateMaxLength(field)}
+        {generateDefaultValue(field)}
         {generateDesc(field)}
 
+        <div className={classes.paramsGrp}>
+          {generateAnalyzerCheckBox(field, fields)}
+          {generateTextMatchCheckBox(field, fields)}
+          {generatePartitionKeyCheckbox(field, fields)}
+          {generateNullableCheckbox(field, fields)}
+        </div>
+
         <IconButton
           onClick={() => {
-            handleAddNewField(index);
+            handleAddNewField(index, field.data_type);
           }}
           classes={{ root: classes.iconBtn }}
           aria-label="add"
@@ -861,17 +907,59 @@ const CreateFields: FC<CreateFieldsProps> = ({
         >
           <AddIcon />
         </IconButton>
+
+        {requiredFields.length !== 2 && (
+          <IconButton
+            onClick={() => {
+              const id = field.id || '';
+              handleRemoveField(id);
+            }}
+            classes={{ root: classes.iconBtn }}
+            aria-label="delete"
+            size="large"
+          >
+            <RemoveIcon />
+          </IconButton>
+        )}
+      </div>
+    );
+  };
+
+  const generateVectorRow = (field: FieldType, index: number) => {
+    return (
+      <div className={`${classes.rowWrapper}`}>
+        {generateFieldName(field)}
+        {getSelector(
+          'vector',
+          `${collectionTrans('vectorType')} `,
+          field.data_type,
+          (value: DataTypeEnum) => changeFields(field.id!, { data_type: value })
+        )}
+
+        {generateDimension(field)}
+        {generateDesc(field)}
+
         <IconButton
-          onClick={() => {
-            const id = field.id || '';
-            handleRemoveField(id);
-          }}
+          onClick={() => handleAddNewField(index, field.data_type)}
           classes={{ root: classes.iconBtn }}
-          aria-label="delete"
+          aria-label="add"
           size="large"
         >
-          <RemoveIcon />
+          <AddIcon />
         </IconButton>
+        {requiredFields.length !== 2 && (
+          <IconButton
+            onClick={() => {
+              const id = field.id || '';
+              handleRemoveField(id);
+            }}
+            classes={{ root: classes.iconBtn }}
+            aria-label="delete"
+            size="large"
+          >
+            <RemoveIcon />
+          </IconButton>
+        )}
       </div>
     );
   };
@@ -879,44 +967,63 @@ const CreateFields: FC<CreateFieldsProps> = ({
   const generateRequiredFieldRow = (
     field: FieldType,
     autoID: boolean,
-    index: number
+    index: number,
+    fields: FieldType[],
+    requiredFields: FieldType[]
   ) => {
     // required type is primaryKey or defaultVector
     if (field.createType === 'primaryKey') {
       return generatePrimaryKeyRow(field, autoID);
     }
-    // use defaultVector as default return type
-    return generateDefaultVectorRow(field, index);
-  };
 
-  const generateOptionalFieldRow = (
-    field: FieldType,
-    index: number,
-    fields: FieldType[]
-  ) => {
-    // optional type is vector or number
-    if (field.createType === 'vector') {
-      return generateVectorRow(field, index);
+    if (field.data_type === DataTypeEnum.VarCharBM25) {
+      return generateFunctionRow(field, index, fields, requiredFields);
+    }
+
+    if (field.createType === 'defaultVector') {
+      return generateDefaultVectorRow(field, index);
     }
 
-    // use number as default createType
-    return generateNonRequiredRow(field, index, fields);
+    // generate other vector rows
+    return generateVectorRow(field, index);
   };
 
   return (
     <>
+      <h4 className={classes.title}>
+        {`${collectionTrans('idAndVectorFields')}(${requiredFields.length})`}
+      </h4>
       {requiredFields.map((field, index) => (
         <Fragment key={field.id}>
-          {generateRequiredFieldRow(field, autoID, index)}
+          {generateRequiredFieldRow(
+            field,
+            autoID,
+            index,
+            fields,
+            requiredFields
+          )}
         </Fragment>
       ))}
-      <div className={classes.optionalWrapper}>
-        {optionalFields.map((field, index) => (
+      <h4 className={classes.title}>
+        {`${collectionTrans('scalarFields')}(${scalarFields.length})`}
+        <IconButton
+          onClick={() => {
+            handleAddNewField(requiredFields.length + 1);
+          }}
+          classes={{ root: classes.iconBtn }}
+          aria-label="add"
+          size="large"
+        >
+          <AddIcon />
+        </IconButton>
+      </h4>
+      <div className={classes.scalarFieldsWrapper}>
+        {scalarFields.map((field, index) => (
           <Fragment key={field.id}>
-            {generateOptionalFieldRow(
+            {generateScalarFieldRow(
               field,
               index + requiredFields.length,
-              optionalFields
+              fields
             )}
           </Fragment>
         ))}

client/src/styles/theme.ts

@@ -1,5 +1,6 @@
 import { PaletteMode } from '@mui/material';
 
+
 const getCommonThemes = (mode: PaletteMode) => ({
   typography: {
     fontFamily: [

client/src/utils/Form.ts

@@ -1,13 +1,13 @@
 import { Option } from '@/components/customSelector/Types';
 import {
   METRIC_TYPES_VALUES,
-  DataTypeEnum,
   SCALAR_INDEX_OPTIONS,
   DataTypeStringEnum,
   INDEX_TYPES_ENUM,
 } from '@/consts';
 import { IForm } from '@/hooks';
 import { IndexType } from '@/pages/databases/collections/schema/Types';
+import { FieldObject } from '@server/types';
 
 interface IInfo {
   [key: string]: any;
@@ -27,7 +27,7 @@ export const formatForm = (info: IInfo): IForm[] => {
 
 export const getMetricOptions = (
   indexType: IndexType,
-  fieldType: DataTypeEnum
+  field: FieldObject
 ): Option[] => {
   const baseFloatOptions = [
     {
@@ -59,19 +59,26 @@ export const getMetricOptions = (
     },
   ];
 
-  switch (fieldType) {
-    case DataTypeEnum.FloatVector:
-    case DataTypeEnum.Float16Vector:
-    case DataTypeEnum.BFloat16Vector:
+  switch (field.data_type) {
+    case DataTypeStringEnum.FloatVector:
+    case DataTypeStringEnum.Float16Vector:
+    case DataTypeStringEnum.BFloat16Vector:
       return baseFloatOptions;
-    case DataTypeEnum.SparseFloatVector:
-      return [
-        {
-          value: METRIC_TYPES_VALUES.IP,
-          label: 'IP',
-        },
-      ];
-    case DataTypeEnum.BinaryVector:
+    case DataTypeStringEnum.SparseFloatVector:
+      return field.is_function_output
+        ? [
+            {
+              value: METRIC_TYPES_VALUES.BM25,
+              label: 'BM25',
+            },
+          ]
+        : [
+            {
+              value: METRIC_TYPES_VALUES.IP,
+              label: 'IP',
+            },
+          ];
+    case DataTypeStringEnum.BinaryVector:
       switch (indexType) {
         case 'BIN_FLAT':
           return [
@@ -95,10 +102,7 @@ export const getMetricOptions = (
   }
 };
 
-export const getScalarIndexOption = (
-  fieldType: DataTypeStringEnum,
-  elementType?: DataTypeStringEnum
-): Option[] => {
+export const getScalarIndexOption = (field: FieldObject): Option[] => {
   // Helper function to check if a type is numeric
   const isNumericType = (type: DataTypeStringEnum): boolean =>
     ['Int8', 'Int16', 'Int32', 'Int64', 'Float', 'Double'].includes(type);
@@ -111,7 +115,7 @@ export const getScalarIndexOption = (
   const options: Option[] = [];
 
   // Add options based on fieldType
-  if (fieldType === DataTypeStringEnum.VarChar) {
+  if (field.data_type === DataTypeStringEnum.VarChar) {
     options.push(
       SCALAR_INDEX_OPTIONS.find(
         opt => opt.value === INDEX_TYPES_ENUM.MARISA_TRIE
@@ -119,21 +123,21 @@ export const getScalarIndexOption = (
     );
   }
 
-  if (isNumericType(fieldType)) {
+  if (isNumericType(field.data_type as DataTypeStringEnum)) {
     options.push(
       SCALAR_INDEX_OPTIONS.find(opt => opt.value === INDEX_TYPES_ENUM.SORT)!
     );
   }
 
   if (
-    fieldType === DataTypeStringEnum.Array &&
-    elementType &&
-    isBitmapSupportedType(elementType)
+    field.data_type === 'Array' &&
+    field.data_type &&
+    isBitmapSupportedType(field.element_type as DataTypeStringEnum)
   ) {
     options.push(
       SCALAR_INDEX_OPTIONS.find(opt => opt.value === INDEX_TYPES_ENUM.BITMAP)!
     );
-  } else if (isBitmapSupportedType(fieldType)) {
+  } else if (isBitmapSupportedType(field.data_type as DataTypeStringEnum)) {
     options.push(
       SCALAR_INDEX_OPTIONS.find(opt => opt.value === INDEX_TYPES_ENUM.BITMAP)!
     );

client/src/utils/Format.ts

@@ -5,6 +5,7 @@ import {
   VectorTypes,
   DataTypeStringEnum,
   DEFAULT_ANALYZER_PARAMS,
+  DataTypeEnum,
 } from '@/consts';
 import {
   CreateFieldType,
@@ -102,16 +103,23 @@ export const checkIsBinarySubstructure = (metricLabel: string): boolean => {
   return metricLabel === 'Superstructure' || metricLabel === 'Substructure';
 };
 
-export const getCreateFieldType = (config: CreateField): CreateFieldType => {
-  if (config.is_primary_key) {
+export const isVectorType = (field: FieldObject): boolean => {
+  return VectorTypes.includes(field.dataType as any);
+}
+
+export const getCreateFieldType = (field: CreateField): CreateFieldType => {
+  if (field.is_primary_key) {
     return 'primaryKey';
   }
 
-  if (config.isDefault) {
+  if (field.isDefault) {
     return 'defaultVector';
   }
 
-  if (VectorTypes.includes(config.data_type)) {
+  if (
+    VectorTypes.includes(field.data_type) ||
+    field.data_type === DataTypeEnum.VarCharBM25
+  ) {
     return 'vector';
   }
 
@@ -284,10 +292,12 @@ export const generateVectorsByField = (field: FieldObject) => {
           ? field.dimension / 8
           : field.dimension;
       return JSON.stringify(generateVector(dim));
-    case 'SparseFloatVector':
-      return transformObjToStr({
-        [Math.floor(Math.random() * 10)]: Math.random(),
-      });
+    case DataTypeStringEnum.SparseFloatVector:
+      return field.is_function_output
+        ? 'fox'
+        : transformObjToStr({
+            [Math.floor(Math.random() * 10)]: Math.random(),
+          });
     default:
       return [1, 2, 3];
   }

client/src/utils/search.ts

@@ -26,7 +26,9 @@ export const buildSearchParams = (searchParams: SearchParams) => {
     if (s.selected) {
       data.push({
         anns_field: s.field.name,
-        data: formatter(s.data),
+        data: s.field.is_function_output
+          ? s.data.replace(/\n/g, '')
+          : formatter(s.data),
         params: s.params,
       });
       weightedParams.push(

server/src/collections/collections.service.ts

@@ -111,6 +111,28 @@ export class CollectionsService {
 
     const vectorFields: FieldObject[] = [];
     const scalarFields: FieldObject[] = [];
+    const functionFields: FieldObject[] = [];
+
+    // assign function to field
+    const fieldMap = new Map(
+      res.schema.fields.map(field => [field.name, field])
+    );
+    res.schema.functions.forEach(fn => {
+      const assignFunction = (fieldName: string) => {
+        const field = fieldMap.get(fieldName);
+        if (field) {
+          field.function = fn;
+        }
+      };
+
+      fn.output_field_names.forEach(assignFunction);
+      fn.input_field_names.forEach(assignFunction);
+    });
+
+    // get function input fields
+    const inputFieldNames = res.schema.functions.reduce((acc, cur) => {
+      return acc.concat(cur.input_field_names);
+    }, []);
 
     // append index info to each field
     res.schema.fields.forEach((field: FieldObject) => {
@@ -119,18 +141,11 @@ export class CollectionsService {
         index => index.field_name === field.name
       ) as IndexObject;
       // add dimension
-      field.dimension =
-        Number(field.type_params.find(item => item.key === 'dim')?.value) || -1;
+      field.dimension = Number(field.dim) || -1;
       // add max capacity
-      field.maxCapacity =
-        Number(
-          field.type_params.find(item => item.key === 'max_capacity')?.value
-        ) || -1;
+      field.maxCapacity = Number(field.max_capacity) || -1;
       // add max length
-      field.maxLength =
-        Number(
-          field.type_params.find(item => item.key === 'max_length')?.value
-        ) || -1;
+      field.maxLength = Number(field.max_length) || -1;
 
       // classify fields
       if (VectorTypes.includes(field.data_type)) {
@@ -142,6 +157,11 @@ export class CollectionsService {
       if (field.is_primary_key) {
         res.schema.primaryField = field;
       }
+
+      // add functionFields if field name included in inputFieldNames
+      if (inputFieldNames.includes(field.name)) {
+        functionFields.push(field);
+      }
     });
 
     // add extra data to schema
@@ -151,6 +171,7 @@ export class CollectionsService {
     );
     res.schema.scalarFields = scalarFields;
     res.schema.vectorFields = vectorFields;
+    res.schema.functionFields = functionFields;
     res.schema.dynamicFields = res.schema.enable_dynamic_field
       ? [
           {

server/src/types/collections.type.ts

@@ -8,9 +8,19 @@ import {
   DescribeCollectionResponse,
   QuerySegmentInfo,
   PersistentSegmentInfo,
+  FunctionType,
 } from '@zilliz/milvus2-sdk-node';
 import { WS_EVENTS, WS_EVENTS_TYPE, LOADING_STATE } from '../utils';
 
+export type FunctionObject = {
+  name: string;
+  description?: string;
+  type: FunctionType;
+  input_field_names: string[];
+  output_field_names?: string[];
+  params: Record<string, any>;
+};
+
 export interface IndexObject extends IndexDescription {
   indexType: string;
   metricType: string;
@@ -22,6 +32,7 @@ export interface FieldObject extends FieldSchema {
   dimension: number;
   maxCapacity: number;
   maxLength: number;
+  function?: FunctionObject;
 }
 
 export interface SchemaObject extends CollectionSchema {
@@ -30,6 +41,7 @@ export interface SchemaObject extends CollectionSchema {
   vectorFields: FieldObject[];
   scalarFields: FieldObject[];
   dynamicFields: FieldObject[];
+  functionFields: FieldObject[];
   hasVectorIndex: boolean;
   enablePartitionKey: boolean;
 }

server/src/utils/Helper.ts

@@ -142,7 +142,7 @@ export const genRow = (
 ) => {
   const result: any = {};
   fields.forEach(field => {
-    if (!field.autoID) {
+    if (!field.autoID && !field.is_function_output) {
       if ((field.nullable || field.default_value) && Math.random() < 0.5) {
         return;
       }