support dynamic field

Signed-off-by: shanghaikid <jiangruiyi@gmail.com>

client/src/consts/Milvus.ts

@@ -3,6 +3,8 @@ import { DataTypeEnum } from '@/pages/collections/Types';
 export const MILVUS_URL =
   ((window as any)._env_ && (window as any)._env_.MILVUS_URL) || '';
 
+export const DYNAMIC_FIELD = `$meta`;
+
 export enum METRIC_TYPES_VALUES {
   L2 = 'L2',
   IP = 'IP',

client/src/http/Collection.ts

@@ -19,7 +19,7 @@ import { LOADING_STATE } from '@/consts';
 
 export class CollectionHttp extends BaseModel implements CollectionView {
   private aliases!: string[];
-  private autoID!: string;
+  private autoID!: boolean;
   private collection_name!: string;
   private description!: string;
   private consistency_level!: string;
@@ -30,6 +30,9 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   private createdTime!: string;
   private schema!: {
     fields: Field[];
+    autoID: boolean;
+    description: string;
+    enable_dynamic_field: boolean;
   };
   private replicas!: Replica[];
 
@@ -233,5 +236,12 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   get _replicas(): Replica[] {
     return this.replicas;
   }
-  
+
+  get _enableDynamicField(): boolean {
+    return this.schema.enable_dynamic_field;
+  }
+
+  get _schema() {
+    return this.schema;
+  }
 }

client/src/i18n/en/collection.ts

@@ -2,7 +2,7 @@ const collectionTrans = {
   noLoadData: 'No Loaded Collection',
   noData: 'No Collection',
 
-  rowCount: 'Approx Entity Count',
+  rowCount: 'Approx Count',
 
   create: 'Collection',
   delete: 'delete',
@@ -22,11 +22,13 @@ const collectionTrans = {
   // table
   id: 'ID',
   name: 'Name',
+  features: 'Features',
   nameTip: 'Collection Name',
   status: 'Status',
   desc: 'Description',
   createdTime: 'Created Time',
   maxLength: 'Max Length',
+  dynmaicSchema: 'Dynamic Schema',
 
   // table tooltip
   aliasInfo: 'Alias can be used as collection name in vector search.',
@@ -38,7 +40,7 @@ const collectionTrans = {
   createTitle: 'Create Collection',
   general: 'General information',
   schema: 'Schema',
-  consistency: 'Consistency Level',
+  consistency: 'Consistency',
   consistencyLevel: 'Consistency Level',
   description: 'Description',
   fieldType: 'Type',
@@ -116,6 +118,11 @@ const collectionTrans = {
   compact: 'Compact',
   compactDialogInfo: `Compaction is a process that optimizes storage and query performance by organizing segments.  <a href='https://milvus.io/blog/2022-2-21-compact.md' target="_blank">Learn more</a><br /><br />  Please note that this operation may take some time to complete, especially for large datasets. We recommend running compaction during periods of lower system activity or during scheduled maintenance to minimize disruption.
     `,
+
+  // column tooltip
+  autoIDTooltip: `A boolean flag that indicates whether the primary key is automatically generated by Milvus.`,
+  dynamicSchemaTooltip: `Dynamic schema enables users to insert entities with new fields into a Milvus collection without modifying the existing schema.`,
+  consistencyLevelTooltip: `Consistency in a distributed database specifically refers to the property that ensures every node or replica has the same view of data when writing or reading data at a given time.`,
 };
 
 export default collectionTrans;

client/src/i18n/en/search.ts

@@ -14,6 +14,7 @@ const searchTrans = {
   vectorValueWarning: 'Vector value should be an array of length {{dimension}}',
   timeTravel: 'Time Travel',
   timeTravelPrefix: 'Data before',
+  dynamicFields: 'Dynamic Fields',
 };
 
 export default searchTrans;

client/src/pages/collections/Collections.tsx

@@ -1,6 +1,6 @@
 import { useCallback, useContext, useEffect, useMemo, useState } from 'react';
 import { Link, useSearchParams } from 'react-router-dom';
-import { makeStyles, Theme } from '@material-ui/core';
+import { makeStyles, Theme, Chip, Tooltip } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
 import Highlighter from 'react-highlight-words';
 import {
@@ -62,6 +62,10 @@ const useStyles = makeStyles((theme: Theme) => ({
     color: theme.palette.primary.main,
     backgroundColor: 'transparent',
   },
+  chip: {
+    color: theme.palette.text.primary,
+    marginRight: theme.spacing(0.5),
+  },
 }));
 
 const Collections = () => {
@@ -137,6 +141,43 @@ const Collections = () => {
             />
           </Link>
         ),
+        features: (
+          <>
+            {v._autoId ? (
+              <Tooltip title={collectionTrans('autoIDTooltip')} placement="top" arrow>
+                <Chip
+                  className={classes.chip}
+                  label={collectionTrans('autoID')}
+                  size="small"
+                />
+              </Tooltip>
+            ) : null}
+            {v._enableDynamicField ? (
+              <Tooltip
+                title={collectionTrans('dynamicSchemaTooltip')}
+                placement="top"
+                arrow
+              >
+                <Chip
+                  className={classes.chip}
+                  label={collectionTrans('dynmaicSchema')}
+                  size="small"
+                />
+              </Tooltip>
+            ) : null}
+            <Tooltip
+              title={collectionTrans('consistencyLevelTooltip')}
+              placement="top"
+              arrow
+            >
+              <Chip
+                className={classes.chip}
+                label={v._consistencyLevel}
+                size="small"
+              />
+            </Tooltip>
+          </>
+        ),
         statusElement: (
           <Status status={v._status} percentage={v._loadedPercentage} />
         ),
@@ -360,6 +401,13 @@ const Collections = () => {
       sortBy: '_status',
       label: collectionTrans('status'),
     },
+    {
+      id: 'features',
+      align: 'left',
+      disablePadding: true,
+      sortBy: '_enableDynamicField',
+      label: collectionTrans('features'),
+    },
     {
       id: '_rowCount',
       align: 'left',
@@ -374,19 +422,19 @@ const Collections = () => {
       ),
     },
 
-    {
-      id: 'consistency_level',
-      align: 'left',
-      disablePadding: false,
-      label: (
-        <span className="flex-center">
-          {collectionTrans('consistencyLevel')}
-          <CustomToolTip title={collectionTrans('consistencyLevelInfo')}>
-            <InfoIcon classes={{ root: classes.icon }} />
-          </CustomToolTip>
-        </span>
-      ),
-    },
+    // {
+    //   id: 'consistency_level',
+    //   align: 'left',
+    //   disablePadding: true,
+    //   label: (
+    //     <span className="flex-center">
+    //       {collectionTrans('consistency')}
+    //       <CustomToolTip title={collectionTrans('consistencyLevelInfo')}>
+    //         <InfoIcon classes={{ root: classes.icon }} />
+    //       </CustomToolTip>
+    //     </span>
+    //   ),
+    // },
 
     {
       id: '_desc',
@@ -480,7 +528,7 @@ const Collections = () => {
   ];
 
   if (!isManaged) {
-    colDefinitions.splice(3, 0, {
+    colDefinitions.splice(4, 0, {
       id: '_aliasElement',
       align: 'left',
       disablePadding: false,

client/src/pages/collections/Types.ts

@@ -15,6 +15,8 @@ export interface CollectionData {
   _consistencyLevel?: string;
   _aliases: string[];
   _replicas: Replica[];
+  _enableDynamicField: boolean;
+  _autoId: boolean;
 }
 
 export interface Replica {

client/src/pages/dialogs/CreateCollectionDialog.tsx

@@ -1,5 +1,10 @@
-import { makeStyles, Theme } from '@material-ui/core';
-import { FC, useContext, useMemo, useState } from 'react';
+import {
+  makeStyles,
+  Theme,
+  Checkbox,
+  FormControlLabel,
+} from '@material-ui/core';
+import { FC, useContext, useMemo, useState, ChangeEvent } from 'react';
 import { useTranslation } from 'react-i18next';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
 import CustomInput from '@/components/customInput/CustomInput';
@@ -26,9 +31,10 @@ const useStyles = makeStyles((theme: Theme) => ({
     alignItems: 'center',
     marginBottom: '16px',
     gap: '8px',
-    '&:nth-last-child(2)': {
+    '&:nth-last-child(3)': {
       flexDirection: 'column',
       alignItems: 'flex-start',
+      marginBottom: '0',
     },
 
     '& legend': {
@@ -64,6 +70,7 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
     collection_name: '',
     description: '',
     autoID: true,
+    enableDynamicField: false,
   });
 
   const [consistencyLevel, setConsistencyLevel] =
@@ -117,6 +124,16 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
     });
   };
 
+  const changeEnableDynamicField = (
+    event: ChangeEvent<any>,
+    value: boolean
+  ) => {
+    setForm({
+      ...form,
+      enableDynamicField: value,
+    });
+  };
+
   const handleInputChange = (key: string, value: string) => {
     setForm(v => ({ ...v, [key]: value }));
   };
@@ -263,6 +280,14 @@ const CreateCollectionDialog: FC<CollectionCreateProps> = ({ onCreate }) => {
             setAutoID={changeIsAutoID}
           />
         </fieldset>
+        <fieldset className={classes.fieldset}>
+          <FormControlLabel
+            checked={form.enableDynamicField}
+            control={<Checkbox />}
+            onChange={changeEnableDynamicField}
+            label="Enable Dynamic Fields"
+          />
+        </fieldset>
 
         <fieldset className={classes.fieldset}>
           <legend>{collectionTrans('consistency')}</legend>

client/src/pages/preview/Preview.tsx

@@ -1,13 +1,17 @@
 import { FC, useEffect, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import AttuGrid from '@/components/grid/Grid';
-import { CollectionHttp, FieldHttp, IndexHttp } from '@/http';
+import { CollectionHttp, IndexHttp } from '@/http';
 import { usePaginationHook, useSearchResult } from '@/hooks';
 import { generateVector } from '@/utils';
-import { INDEX_CONFIG, DEFAULT_SEARCH_PARAM_VALUE_MAP } from '@/consts';
+import {
+  INDEX_CONFIG,
+  DEFAULT_SEARCH_PARAM_VALUE_MAP,
+  DYNAMIC_FIELD,
+} from '@/consts';
 import { ToolBarConfig } from '@/components/grid/Types';
 import CustomToolBar from '@/components/grid/ToolBar';
-import { DataTypeEnum } from '@/pages/collections/Types';
+import { DataTypeEnum, DataTypeStringEnum } from '@/pages/collections/Types';
 import { getQueryStyles } from '../query/Styles';
 
 const Preview: FC<{
@@ -18,6 +22,7 @@ const Preview: FC<{
   const [queryResult, setQueryResult] = useState<any>();
   const [primaryKey, setPrimaryKey] = useState<string>('');
   const { t: collectionTrans } = useTranslation('collection');
+  const { t: searchTrans } = useTranslation('search');
 
   const classes = getQueryStyles();
 
@@ -39,12 +44,24 @@ const Preview: FC<{
 
   const loadData = async (collectionName: string) => {
     // get schema list
-    const schemaList = await FieldHttp.getFields(collectionName);
-    const nameList = schemaList.map(v => ({
+    const collection = new CollectionHttp(
+      await CollectionHttp.getCollection(collectionName)
+    );
+
+    const schemaList = collection._fields;
+    let nameList = schemaList.map(v => ({
       name: v.name,
       type: v.data_type,
     }));
 
+    // if the dynamic field is enabled, we add $meta column in the grid
+    if (collection._enableDynamicField) {
+      nameList.push({
+        name: DYNAMIC_FIELD,
+        type: DataTypeStringEnum.JSON,
+      });
+    }
+
     const id = schemaList.find(v => v._isPrimaryKey === true);
     const primaryKey = id?._fieldName || '';
     const delimiter = id?.data_type === 'Int64' ? '' : '"';
@@ -99,7 +116,7 @@ const Preview: FC<{
       // query by random id
       const res = await CollectionHttp.queryData(collectionName, {
         expr: expr,
-        output_fields: nameList.map(i => i.name),
+        output_fields: [...nameList.map(i => i.name)],
       });
 
       const result = res.data;
@@ -137,7 +154,8 @@ const Preview: FC<{
           id: i.name,
           align: 'left',
           disablePadding: false,
-          label: i.name,
+          label:
+            i.name === DYNAMIC_FIELD ? searchTrans('dynamicFields') : i.name,
         }))}
         primaryKey={primaryKey}
         openCheckBox={false}

client/src/pages/query/Query.tsx

@@ -4,7 +4,7 @@ import { useTranslation } from 'react-i18next';
 import { saveAs } from 'file-saver';
 import { Parser } from '@json2csv/plainjs';
 import { rootContext } from '@/context';
-import { CollectionHttp, FieldHttp } from '@/http';
+import { CollectionHttp } from '@/http';
 import { usePaginationHook, useSearchResult } from '@/hooks';
 import EmptyCard from '@/components/cards/EmptyCard';
 import icons from '@/components/icons/Icons';
@@ -19,6 +19,7 @@ import CustomToolBar from '@/components/grid/ToolBar';
 import { DataTypeStringEnum } from '../collections/Types';
 import { getLabelDisplayedRows } from '../search/Utils';
 import { getQueryStyles } from './Styles';
+import { DYNAMIC_FIELD } from '@/consts';
 
 const Query: FC<{
   collectionName: string;
@@ -68,11 +69,24 @@ const Query: FC<{
   };
 
   const getFields = async (collectionName: string) => {
-    const schemaList = await FieldHttp.getFields(collectionName);
+    const collection = new CollectionHttp(
+      await CollectionHttp.getCollection(collectionName)
+    );
+    const schemaList = collection._fields;
+
     const nameList = schemaList.map(v => ({
       name: v.name,
       type: v.data_type,
     }));
+
+    // if the dynamic field is enabled, we add $meta column in the grid
+    if (collection._enableDynamicField) {
+      nameList.push({
+        name: DYNAMIC_FIELD,
+        type: DataTypeStringEnum.JSON,
+      });
+    }
+
     const primaryKey = schemaList.find(v => v._isPrimaryKey === true)!;
     setPrimaryKey({ value: primaryKey['name'], type: primaryKey['data_type'] });
 
@@ -266,7 +280,8 @@ const Query: FC<{
             id: i.name,
             align: 'left',
             disablePadding: false,
-            label: i.name,
+            label:
+              i.name === DYNAMIC_FIELD ? searchTrans('dynamicFields') : i.name,
           }))}
           primaryKey={primaryKey.value}
           openCheckBox={true}

client/src/pages/search/VectorSearch.tsx

@@ -27,7 +27,11 @@ import {
   cloneObj,
   generateVector,
 } from '@/utils';
-import { LOADING_STATE, DEFAULT_METRIC_VALUE_MAP } from '@/consts';
+import {
+  LOADING_STATE,
+  DEFAULT_METRIC_VALUE_MAP,
+  DYNAMIC_FIELD,
+} from '@/consts';
 import { getLabelDisplayedRows } from './Utils';
 import SearchParams from './SearchParams';
 import { getVectorSearchStyles } from './Styles';
@@ -97,14 +101,25 @@ const VectorSearch = () => {
   );
 
   const outputFields: string[] = useMemo(() => {
-    const fields =
-      collections.find(c => c._name === selectedCollection)?._fields || [];
+    const s = collections.find(c => c._name === selectedCollection);
+
+    if (!s) {
+      return [];
+    }
+
+    const fields = s._fields || [];
+
     // vector field can't be output fields
     const invalidTypes = ['BinaryVector', 'FloatVector'];
     const nonVectorFields = fields.filter(
       field => !invalidTypes.includes(field._fieldType)
     );
-    return nonVectorFields.map(f => f._fieldName);
+
+    const _outputFields = nonVectorFields.map(f => f._fieldName);
+    if (s._enableDynamicField) {
+      _outputFields.push(DYNAMIC_FIELD);
+    }
+    return _outputFields;
   }, [selectedCollection, collections]);
 
   const primaryKeyField = useMemo(() => {
@@ -139,7 +154,7 @@ const VectorSearch = () => {
             id: key,
             align: 'left',
             disablePadding: false,
-            label: key,
+            label: key === DYNAMIC_FIELD ? searchTrans('dynamicFields') : key,
             needCopy: primaryKeyField === key,
           }))
       : [];

server/src/collections/collections.service.ts

@@ -25,7 +25,7 @@ import { throwErrorFromSDK, findKeyValue, genRows, ROW_COUNT } from '../utils';
 import { QueryDto, ImportSampleDto, GetReplicasDto } from './dto';
 
 export class CollectionsService {
-  constructor(private milvusService: MilvusService) { }
+  constructor(private milvusService: MilvusService) {}
 
   async getCollections(data?: ShowCollectionsReq) {
     const res = await this.milvusService.client.showCollections(data);
@@ -187,8 +187,8 @@ export class CollectionsService {
         try {
           replicas = loadCollection
             ? await this.getReplicas({
-              collectionID: collectionInfo.collectionID,
-            })
+                collectionID: collectionInfo.collectionID,
+              })
             : replicas;
         } catch (e) {
           console.log('ignore getReplica');
@@ -287,7 +287,8 @@ export class CollectionsService {
     const collectionInfo = await this.describeCollection({ collection_name });
     const fields_data = genRows(
       collectionInfo.schema.fields,
-      parseInt(size, 10)
+      parseInt(size, 10),
+      collectionInfo.schema.enable_dynamic_field
     );
 
     return await this.insert({ collection_name, fields_data });

server/src/collections/dto.ts

@@ -29,6 +29,10 @@ export class CreateCollectionDto {
   @IsOptional()
   readonly autoID: boolean;
 
+  @IsBoolean()
+  @IsOptional()
+  readonly enableDynamicField: boolean;
+
   @IsArray()
   @ArrayNotEmpty()
   readonly fields: FieldType[];

server/src/utils/Helper.ts

@@ -6,12 +6,15 @@ import {
 export const findKeyValue = (obj: KeyValuePair[], key: string) =>
   obj.find(v => v.key === key)?.value;
 
+export const makeDynamicBool = () => Math.random() > 0.5;
+export const makeRandomInt = () => Math.floor(Math.random() * 127);
+
 export const genDataByType = ({ data_type, type_params }: FieldSchema) => {
   switch (data_type) {
     case 'Bool':
-      return Math.random() > 0.5;
+      return makeDynamicBool();
     case 'Int8':
-      return Math.floor(Math.random() * 127);
+      return makeRandomInt();
     case 'Int16':
       return Math.floor(Math.random() * 32767);
     case 'Int32':
@@ -23,8 +26,8 @@ export const genDataByType = ({ data_type, type_params }: FieldSchema) => {
         Math.random()
       );
     case 'BinaryVector':
-      return Array.from({ length: (type_params as any)[0].value / 8 }).map(() =>
-        Math.random() > 0.5 ? 1 : 0
+      return Array.from({ length: (type_params as any)[0].value / 8 }).map(
+        () => (Math.random() > 0.5 ? 1 : 0)
       );
     case 'VarChar':
       return makeRandomId((type_params as any)[0].value);
@@ -33,20 +36,33 @@ export const genDataByType = ({ data_type, type_params }: FieldSchema) => {
   }
 };
 
-export const genRow = (fields: FieldSchema[]) => {
+export const genRow = (
+  fields: FieldSchema[],
+  enableDynamicField: boolean = false
+) => {
   const result: any = {};
   fields.forEach(field => {
     if (!field.autoID) {
       result[field.name] = genDataByType(field);
     }
   });
+
+  if (enableDynamicField) {
+    result.dynamicBool = makeDynamicBool();
+    result.dynamicInt = makeRandomInt();
+    result.dynamicJSON = makeRandomJSON();
+  }
   return result;
 };
 
-export const genRows = (fields: FieldSchema[], size: number) => {
+export const genRows = (
+  fields: FieldSchema[],
+  size: number,
+  enableDynamicField: boolean = false
+) => {
   const result = [];
   for (let i = 0; i < size; i++) {
-    result[i] = genRow(fields);
+    result[i] = genRow(fields, enableDynamicField);
   }
   return result;
 };