Merge pull request #274 from zilliztech/dynamic_field

support dynamic field

client/src/consts/Milvus.ts

@@ -3,6 +3,8 @@ import { DataTypeEnum } from '@/pages/collections/Types';
 export const MILVUS_URL =
   ((window as any)._env_ && (window as any)._env_.MILVUS_URL) || '';
 
+export const DYNAMIC_FIELD = `$meta`;
+
 export enum METRIC_TYPES_VALUES {
   L2 = 'L2',
   IP = 'IP',

client/src/http/BaseModel.ts

@@ -50,7 +50,7 @@ export default class BaseModel {
     } as any;
     if (timeout) httpConfig.timeout = timeout;
     const res = await http(httpConfig);
-    return res.data.data;
+    return new this(res.data.data || {});
   }
 
   /**

client/src/http/Collection.ts

@@ -19,7 +19,7 @@ import { LOADING_STATE } from '@/consts';
 
 export class CollectionHttp extends BaseModel implements CollectionView {
   private aliases!: string[];
-  private autoID!: string;
+  private autoID!: boolean;
   private collection_name!: string;
   private description!: string;
   private consistency_level!: string;
@@ -30,6 +30,9 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   private createdTime!: string;
   private schema!: {
     fields: Field[];
+    autoID: boolean;
+    description: string;
+    enable_dynamic_field: boolean;
   };
   private replicas!: Replica[];
 
@@ -52,7 +55,7 @@ export class CollectionHttp extends BaseModel implements CollectionView {
     return super.search({
       path: `${this.COLLECTIONS_URL}/${name}`,
       params: {},
-    });
+    }) as Promise<CollectionHttp>;
   }
 
   static createCollection(data: any) {
@@ -233,5 +236,12 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   get _replicas(): Replica[] {
     return this.replicas;
   }
-  
+
+  get _enableDynamicField(): boolean {
+    return this.schema.enable_dynamic_field;
+  }
+
+  get _schema() {
+    return this.schema;
+  }
 }

client/src/i18n/en/collection.ts

@@ -2,7 +2,7 @@ const collectionTrans = {
   noLoadData: 'No Loaded Collection',
   noData: 'No Collection',
 
-  rowCount: 'Approx Entity Count',
+  rowCount: 'Approx Count',
 
   create: 'Collection',
   delete: 'delete',
@@ -22,11 +22,13 @@ const collectionTrans = {
   // table
   id: 'ID',
   name: 'Name',
+  features: 'Features',
   nameTip: 'Collection Name',
   status: 'Status',
   desc: 'Description',
   createdTime: 'Created Time',
   maxLength: 'Max Length',
+  dynmaicSchema: 'Dynamic Schema',
 
   // table tooltip
   aliasInfo: 'Alias can be used as collection name in vector search.',
@@ -38,7 +40,7 @@ const collectionTrans = {
   createTitle: 'Create Collection',
   general: 'General information',
   schema: 'Schema',
-  consistency: 'Consistency Level',
+  consistency: 'Consistency',
   consistencyLevel: 'Consistency Level',
   description: 'Description',
   fieldType: 'Type',
@@ -63,6 +65,7 @@ const collectionTrans = {
   partitionKey: 'Partition Key',
   partitionKeyTooltip:
     ' Milvus will store entities in a partition according to the values in the partition key field. Only one Int64 or VarChar field is supported.',
+  enableDynamicSchema: 'Enable Dynamic Schema',
 
   // load dialog
   loadTitle: 'Load Collection',
@@ -116,6 +119,15 @@ const collectionTrans = {
   compact: 'Compact',
   compactDialogInfo: `Compaction is a process that optimizes storage and query performance by organizing segments.  <a href='https://milvus.io/blog/2022-2-21-compact.md' target="_blank">Learn more</a><br /><br />  Please note that this operation may take some time to complete, especially for large datasets. We recommend running compaction during periods of lower system activity or during scheduled maintenance to minimize disruption.
     `,
+
+  // column tooltip
+  autoIDTooltip: `The values of the primary key column are automatically generated by Milvus.`,
+  dynamicSchemaTooltip: `Dynamic schema enables users to insert entities with new fields into a Milvus collection without modifying the existing schema.`,
+  consistencyLevelTooltip: `Consistency in a distributed database specifically refers to the property that ensures every node or replica has the same view of data when writing or reading data at a given time.`,
+  consistencyBoundedTooltip: `It allows data inconsistency during a certain period of time`,
+  consistencyStrongTooltip: `It ensures that users can read the latest version of data.`,
+  consistencySessionTooltip: `It ensures that all data writes can be immediately perceived in reads during the same session.`,
+  consistencyEventuallyTooltip: `There is no guaranteed order of reads and writes, and replicas eventually converge to the same state given that no further write operations are done.`,
 };
 
 export default collectionTrans;

client/src/i18n/en/search.ts

@@ -14,6 +14,7 @@ const searchTrans = {
   vectorValueWarning: 'Vector value should be an array of length {{dimension}}',
   timeTravel: 'Time Travel',
   timeTravelPrefix: 'Data before',
+  dynamicFields: 'Dynamic Fields',
 };
 
 export default searchTrans;

client/src/pages/collections/Collections.tsx

@@ -1,6 +1,6 @@
 import { useCallback, useContext, useEffect, useMemo, useState } from 'react';
 import { Link, useSearchParams } from 'react-router-dom';
-import { makeStyles, Theme } from '@material-ui/core';
+import { makeStyles, Theme, Chip, Tooltip } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
 import Highlighter from 'react-highlight-words';
 import {
@@ -62,6 +62,10 @@ const useStyles = makeStyles((theme: Theme) => ({
     color: theme.palette.primary.main,
     backgroundColor: 'transparent',
   },
+  chip: {
+    color: theme.palette.text.primary,
+    marginRight: theme.spacing(0.5),
+  },
 }));
 
 const Collections = () => {
@@ -90,6 +94,13 @@ const Collections = () => {
   const InfoIcon = icons.info;
   const SourceIcon = icons.source;
 
+  const consistencyTooltipsMap: Record<string, string> = {
+    Strong: collectionTrans('consistencyStrongTooltip'),
+    Bounded: collectionTrans('consistencyBoundedTooltip'),
+    Session: collectionTrans('consistencySessionTooltip'),
+    Eventually: collectionTrans('consistencyEventuallyTooltip'),
+  };
+
   const fetchData = useCallback(async () => {
     try {
       setLoading(true);
@@ -137,6 +148,47 @@ const Collections = () => {
             />
           </Link>
         ),
+        features: (
+          <>
+            {v._autoId ? (
+              <Tooltip
+                title={collectionTrans('autoIDTooltip')}
+                placement="top"
+                arrow
+              >
+                <Chip
+                  className={classes.chip}
+                  label={collectionTrans('autoID')}
+                  size="small"
+                />
+              </Tooltip>
+            ) : null}
+            {v._enableDynamicField ? (
+              <Tooltip
+                title={collectionTrans('dynamicSchemaTooltip')}
+                placement="top"
+                arrow
+              >
+                <Chip
+                  className={classes.chip}
+                  label={collectionTrans('dynmaicSchema')}
+                  size="small"
+                />
+              </Tooltip>
+            ) : null}
+            <Tooltip
+              title={consistencyTooltipsMap[v._consistencyLevel]}
+              placement="top"
+              arrow
+            >
+              <Chip
+                className={classes.chip}
+                label={v._consistencyLevel}
+                size="small"
+              />
+            </Tooltip>
+          </>
+        ),
         statusElement: (
           <Status status={v._status} percentage={v._loadedPercentage} />
         ),
@@ -360,6 +412,13 @@ const Collections = () => {
       sortBy: '_status',
       label: collectionTrans('status'),
     },
+    {
+      id: 'features',
+      align: 'left',
+      disablePadding: true,
+      sortBy: '_enableDynamicField',
+      label: collectionTrans('features'),
+    },
     {
       id: '_rowCount',
       align: 'left',
@@ -374,19 +433,19 @@ const Collections = () => {
       ),
     },
 
-    {
-      id: 'consistency_level',
-      align: 'left',
-      disablePadding: false,
-      label: (
-        <span className="flex-center">
-          {collectionTrans('consistencyLevel')}
-          <CustomToolTip title={collectionTrans('consistencyLevelInfo')}>
-            <InfoIcon classes={{ root: classes.icon }} />
-          </CustomToolTip>
-        </span>
-      ),
-    },
+    // {
+    //   id: 'consistency_level',
+    //   align: 'left',
+    //   disablePadding: true,
+    //   label: (
+    //     <span className="flex-center">
+    //       {collectionTrans('consistency')}
+    //       <CustomToolTip title={collectionTrans('consistencyLevelInfo')}>
+    //         <InfoIcon classes={{ root: classes.icon }} />
+    //       </CustomToolTip>
+    //     </span>
+    //   ),
+    // },
 
     {
       id: '_desc',
@@ -480,7 +539,7 @@ const Collections = () => {
   ];
 
   if (!isManaged) {
-    colDefinitions.splice(3, 0, {
+    colDefinitions.splice(4, 0, {
       id: '_aliasElement',
       align: 'left',
       disablePadding: false,

client/src/pages/collections/Constants.ts

@@ -18,10 +18,6 @@ export const CONSISTENCY_LEVEL_OPTIONS: KeyValuePair[] = [
     label: 'Eventually',
     value: ConsistencyLevelEnum.Eventually,
   },
-  {
-    label: 'Customized',
-    value: ConsistencyLevelEnum.Customized,
-  },
 ];
 
 export const VECTOR_FIELDS_OPTIONS: KeyValuePair[] = [

client/src/pages/collections/Types.ts

@@ -12,9 +12,11 @@ export interface CollectionData {
   _desc: string;
   _indexState: ChildrenStatusType;
   _fields?: FieldData[];
-  _consistencyLevel?: string;
+  _consistencyLevel: string;
   _aliases: string[];
   _replicas: Replica[];
+  _enableDynamicField: boolean;
+  _autoId: boolean;
 }
 
 export interface Replica {

client/src/pages/dialogs/CreateCollectionDialog.tsx

@@ -1,5 +1,10 @@
-import { makeStyles, Theme } from '@material-ui/core';
-import { FC, useContext, useMemo, useState } from 'react';
+import {
+  makeStyles,
+  Theme,
+  Checkbox,
+  FormControlLabel,
+} from '@material-ui/core';
+import { FC, useContext, useMemo, useState, ChangeEvent } from 'react';
 import { useTranslation } from 'react-i18next';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
 import CustomInput from '@/components/customInput/CustomInput';
@@ -25,10 +30,10 @@ const useStyles = makeStyles((theme: Theme) => ({
     display: 'flex',
     alignItems: 'center',
     marginBottom: '16px',
-    gap: '8px',
-    '&:nth-last-child(2)': {
+    '&:nth-last-child(3)': {
       flexDirection: 'column',
       alignItems: 'flex-start',
+      marginBottom: '0',
     },
 
     '& legend': {
@@ -64,6 +69,7 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
     collection_name: '',
     description: '',
     autoID: true,
+    enableDynamicField: false,
   });
 
   const [consistencyLevel, setConsistencyLevel] =
@@ -117,6 +123,16 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
     });
   };
 
+  const changeEnableDynamicField = (
+    event: ChangeEvent<any>,
+    value: boolean
+  ) => {
+    setForm({
+      ...form,
+      enableDynamicField: value,
+    });
+  };
+
   const handleInputChange = (key: string, value: string) => {
     setForm(v => ({ ...v, [key]: value }));
   };
@@ -263,6 +279,14 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
             setAutoID={changeIsAutoID}
           />
         </fieldset>
+        <fieldset className={classes.fieldset}>
+          <FormControlLabel
+            checked={form.enableDynamicField}
+            control={<Checkbox />}
+            onChange={changeEnableDynamicField}
+            label={collectionTrans('enableDynamicSchema')}
+          />
+        </fieldset>
 
         <fieldset className={classes.fieldset}>
           <legend>{collectionTrans('consistency')}</legend>

client/src/pages/preview/Preview.tsx

@@ -1,13 +1,17 @@
 import { FC, useEffect, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import AttuGrid from '@/components/grid/Grid';
-import { CollectionHttp, FieldHttp, IndexHttp } from '@/http';
+import { CollectionHttp, IndexHttp } from '@/http';
 import { usePaginationHook, useSearchResult } from '@/hooks';
 import { generateVector } from '@/utils';
-import { INDEX_CONFIG, DEFAULT_SEARCH_PARAM_VALUE_MAP } from '@/consts';
+import {
+  INDEX_CONFIG,
+  DEFAULT_SEARCH_PARAM_VALUE_MAP,
+  DYNAMIC_FIELD,
+} from '@/consts';
 import { ToolBarConfig } from '@/components/grid/Types';
 import CustomToolBar from '@/components/grid/ToolBar';
-import { DataTypeEnum } from '@/pages/collections/Types';
+import { DataTypeEnum, DataTypeStringEnum } from '@/pages/collections/Types';
 import { getQueryStyles } from '../query/Styles';
 
 const Preview: FC<{
@@ -18,6 +22,7 @@ const Preview: FC<{
   const [queryResult, setQueryResult] = useState<any>();
   const [primaryKey, setPrimaryKey] = useState<string>('');
   const { t: collectionTrans } = useTranslation('collection');
+  const { t: searchTrans } = useTranslation('search');
 
   const classes = getQueryStyles();
 
@@ -39,12 +44,22 @@ const Preview: FC<{
 
   const loadData = async (collectionName: string) => {
     // get schema list
-    const schemaList = await FieldHttp.getFields(collectionName);
-    const nameList = schemaList.map(v => ({
+    const collection = await CollectionHttp.getCollection(collectionName);
+
+    const schemaList = collection._fields!;
+    let nameList = schemaList.map(v => ({
       name: v.name,
       type: v.data_type,
     }));
 
+    // if the dynamic field is enabled, we add $meta column in the grid
+    if (collection._enableDynamicField) {
+      nameList.push({
+        name: DYNAMIC_FIELD,
+        type: DataTypeStringEnum.JSON,
+      });
+    }
+
     const id = schemaList.find(v => v._isPrimaryKey === true);
     const primaryKey = id?._fieldName || '';
     const delimiter = id?.data_type === 'Int64' ? '' : '"';
@@ -99,7 +114,7 @@ const Preview: FC<{
       // query by random id
       const res = await CollectionHttp.queryData(collectionName, {
         expr: expr,
-        output_fields: nameList.map(i => i.name),
+        output_fields: [...nameList.map(i => i.name)],
       });
 
       const result = res.data;
@@ -137,7 +152,8 @@ const Preview: FC<{
           id: i.name,
           align: 'left',
           disablePadding: false,
-          label: i.name,
+          label:
+            i.name === DYNAMIC_FIELD ? searchTrans('dynamicFields') : i.name,
         }))}
         primaryKey={primaryKey}
         openCheckBox={false}

client/src/pages/query/Query.tsx

@@ -4,7 +4,7 @@ import { useTranslation } from 'react-i18next';
 import { saveAs } from 'file-saver';
 import { Parser } from '@json2csv/plainjs';
 import { rootContext } from '@/context';
-import { CollectionHttp, FieldHttp } from '@/http';
+import { CollectionHttp } from '@/http';
 import { usePaginationHook, useSearchResult } from '@/hooks';
 import EmptyCard from '@/components/cards/EmptyCard';
 import icons from '@/components/icons/Icons';
@@ -19,6 +19,7 @@ import CustomToolBar from '@/components/grid/ToolBar';
 import { DataTypeStringEnum } from '../collections/Types';
 import { getLabelDisplayedRows } from '../search/Utils';
 import { getQueryStyles } from './Styles';
+import { DYNAMIC_FIELD } from '@/consts';
 
 const Query: FC<{
   collectionName: string;
@@ -68,11 +69,22 @@ const Query: FC<{
   };
 
   const getFields = async (collectionName: string) => {
-    const schemaList = await FieldHttp.getFields(collectionName);
+    const collection = await CollectionHttp.getCollection(collectionName);
+    const schemaList = collection._fields;
+
     const nameList = schemaList.map(v => ({
       name: v.name,
       type: v.data_type,
     }));
+
+    // if the dynamic field is enabled, we add $meta column in the grid
+    if (collection._enableDynamicField) {
+      nameList.push({
+        name: DYNAMIC_FIELD,
+        type: DataTypeStringEnum.JSON,
+      });
+    }
+
     const primaryKey = schemaList.find(v => v._isPrimaryKey === true)!;
     setPrimaryKey({ value: primaryKey['name'], type: primaryKey['data_type'] });
 
@@ -266,7 +278,8 @@ const Query: FC<{
             id: i.name,
             align: 'left',
             disablePadding: false,
-            label: i.name,
+            label:
+              i.name === DYNAMIC_FIELD ? searchTrans('dynamicFields') : i.name,
           }))}
           primaryKey={primaryKey.value}
           openCheckBox={true}

client/src/pages/search/VectorSearch.tsx

@@ -27,7 +27,11 @@ import {
   cloneObj,
   generateVector,
 } from '@/utils';
-import { LOADING_STATE, DEFAULT_METRIC_VALUE_MAP } from '@/consts';
+import {
+  LOADING_STATE,
+  DEFAULT_METRIC_VALUE_MAP,
+  DYNAMIC_FIELD,
+} from '@/consts';
 import { getLabelDisplayedRows } from './Utils';
 import SearchParams from './SearchParams';
 import { getVectorSearchStyles } from './Styles';
@@ -97,14 +101,25 @@ const VectorSearch = () => {
   );
 
   const outputFields: string[] = useMemo(() => {
-    const fields =
-      collections.find(c => c._name === selectedCollection)?._fields || [];
+    const s = collections.find(c => c._name === selectedCollection);
+
+    if (!s) {
+      return [];
+    }
+
+    const fields = s._fields || [];
+
     // vector field can't be output fields
     const invalidTypes = ['BinaryVector', 'FloatVector'];
     const nonVectorFields = fields.filter(
       field => !invalidTypes.includes(field._fieldType)
     );
-    return nonVectorFields.map(f => f._fieldName);
+
+    const _outputFields = nonVectorFields.map(f => f._fieldName);
+    if (s._enableDynamicField) {
+      _outputFields.push(DYNAMIC_FIELD);
+    }
+    return _outputFields;
   }, [selectedCollection, collections]);
 
   const primaryKeyField = useMemo(() => {
@@ -139,7 +154,7 @@ const VectorSearch = () => {
             id: key,
             align: 'left',
             disablePadding: false,
-            label: key,
+            label: key === DYNAMIC_FIELD ? searchTrans('dynamicFields') : key,
             needCopy: primaryKeyField === key,
           }))
       : [];

server/src/collections/collections.service.ts

@@ -25,7 +25,7 @@ import { throwErrorFromSDK, findKeyValue, genRows, ROW_COUNT } from '../utils';
 import { QueryDto, ImportSampleDto, GetReplicasDto } from './dto';
 
 export class CollectionsService {
-  constructor(private milvusService: MilvusService) { }
+  constructor(private milvusService: MilvusService) {}
 
   async getCollections(data?: ShowCollectionsReq) {
     const res = await this.milvusService.client.showCollections(data);
@@ -187,8 +187,8 @@ export class CollectionsService {
         try {
           replicas = loadCollection
             ? await this.getReplicas({
-              collectionID: collectionInfo.collectionID,
-            })
+                collectionID: collectionInfo.collectionID,
+              })
             : replicas;
         } catch (e) {
           console.log('ignore getReplica');
@@ -287,7 +287,8 @@ export class CollectionsService {
     const collectionInfo = await this.describeCollection({ collection_name });
     const fields_data = genRows(
       collectionInfo.schema.fields,
-      parseInt(size, 10)
+      parseInt(size, 10),
+      collectionInfo.schema.enable_dynamic_field
     );
 
     return await this.insert({ collection_name, fields_data });

server/src/collections/dto.ts

@@ -29,6 +29,10 @@ export class CreateCollectionDto {
   @IsOptional()
   readonly autoID: boolean;
 
+  @IsBoolean()
+  @IsOptional()
+  readonly enableDynamicField: boolean;
+
   @IsArray()
   @ArrayNotEmpty()
   readonly fields: FieldType[];

server/src/utils/Helper.ts

@@ -6,12 +6,15 @@ import {
 export const findKeyValue = (obj: KeyValuePair[], key: string) =>
   obj.find(v => v.key === key)?.value;
 
+export const makeDynamicBool = () => Math.random() > 0.5;
+export const makeRandomInt = () => Math.floor(Math.random() * 127);
+
 export const genDataByType = ({ data_type, type_params }: FieldSchema) => {
   switch (data_type) {
     case 'Bool':
-      return Math.random() > 0.5;
+      return makeDynamicBool();
     case 'Int8':
-      return Math.floor(Math.random() * 127);
+      return makeRandomInt();
     case 'Int16':
       return Math.floor(Math.random() * 32767);
     case 'Int32':
@@ -23,8 +26,8 @@ export const genDataByType = ({ data_type, type_params }: FieldSchema) => {
         Math.random()
       );
     case 'BinaryVector':
-      return Array.from({ length: (type_params as any)[0].value / 8 }).map(() =>
-        Math.random() > 0.5 ? 1 : 0
+      return Array.from({ length: (type_params as any)[0].value / 8 }).map(
+        () => (Math.random() > 0.5 ? 1 : 0)
       );
     case 'VarChar':
       return makeRandomId((type_params as any)[0].value);
@@ -33,20 +36,33 @@ export const genDataByType = ({ data_type, type_params }: FieldSchema) => {
   }
 };
 
-export const genRow = (fields: FieldSchema[]) => {
+export const genRow = (
+  fields: FieldSchema[],
+  enableDynamicField: boolean = false
+) => {
   const result: any = {};
   fields.forEach(field => {
     if (!field.autoID) {
       result[field.name] = genDataByType(field);
     }
   });
+
+  if (enableDynamicField) {
+    result.dynamicBool = makeDynamicBool();
+    result.dynamicInt = makeRandomInt();
+    result.dynamicJSON = makeRandomJSON();
+  }
   return result;
 };
 
-export const genRows = (fields: FieldSchema[], size: number) => {
+export const genRows = (
+  fields: FieldSchema[],
+  size: number,
+  enableDynamicField: boolean = false
+) => {
   const result = [];
   for (let i = 0; i < size; i++) {
-    result[i] = genRow(fields);
+    result[i] = genRow(fields, enableDynamicField);
   }
   return result;
 };