add collection insert

client/src/components/insert/Container.tsx

@@ -1,5 +1,13 @@
 import { makeStyles, Theme } from '@material-ui/core';
-import { FC, ReactElement, useContext, useMemo, useState } from 'react';
+import {
+  FC,
+  ReactElement,
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useState,
+} from 'react';
 import { useTranslation } from 'react-i18next';
 import DialogTemplate from '../customDialog/DialogTemplate';
 import icons from '../icons/Icons';
@@ -14,6 +22,8 @@ import {
 } from './Types';
 import { Option } from '../customSelector/Types';
 import { parse } from 'papaparse';
+import { PartitionHttp } from '../../http/Partition';
+import { combineHeadsAndData } from '../../utils/Insert';
 
 const getStyles = makeStyles((theme: Theme) => ({
   icon: {
@@ -23,53 +33,19 @@ const getStyles = makeStyles((theme: Theme) => ({
 
 /**
  * this component contains processes during insert
- * all datas and methods passed in as props, no interactions with server done in it
+ * including import, preview and status
  */
 
 const InsertContainer: FC<InsertContentProps> = ({
-  collections,
-  partitions,
-
-  /**
-   * every time selected collection change,
-   * we need to call handleSelectedCollectionChange function to update partitions and schema data,
-   */
+  collections = [],
   defaultSelectedCollection,
-  handleSelectedCollectionChange,
-
   defaultSelectedPartition,
 
-  schema,
+  schema = [],
   handleInsert,
 }) => {
   const classes = getStyles();
 
-  // props children component needed:
-  const collectionOptions: Option[] = useMemo(
-    () =>
-      collections.map(c => ({
-        label: c._name,
-        value: c._name,
-      })),
-    [collections]
-  );
-  const partitionOptions: Option[] = useMemo(
-    () =>
-      partitions.map(p => ({
-        label: p._name,
-        value: p._name,
-      })),
-    [partitions]
-  );
-  const schemaOptions: Option[] = useMemo(
-    () =>
-      schema.map(s => ({
-        label: s._fieldName,
-        value: s._fieldId,
-      })),
-    [schema]
-  );
-
   const { t: insertTrans } = useTranslation('insert');
   const { t: btnTrans } = useTranslation('btn');
   const { handleCloseDialog, openSnackBar } = useContext(rootContext);
@@ -96,6 +72,38 @@ const InsertContainer: FC<InsertContentProps> = ({
   // uploaded csv data (type: string)
   const [csvData, setCsvData] = useState<any[]>([]);
 
+  // handle changed table heads
+  const [tableHeads, setTableHeads] = useState<string[]>([]);
+
+  const previewData = useMemo(() => {
+    // we only show top 4 results of uploaded csv data
+    const end = isContainFieldNames ? 5 : 4;
+    return csvData.slice(0, end);
+  }, [csvData, isContainFieldNames]);
+
+  useEffect(() => {
+    const heads = isContainFieldNames
+      ? previewData[0]
+      : new Array(previewData[0].length).fill('');
+
+    setTableHeads(heads);
+  }, [previewData, isContainFieldNames]);
+
+  const fetchPartition = useCallback(async () => {
+    if (collectionValue) {
+      const partitions = await PartitionHttp.getPartitions(collectionValue);
+      const partitionOptions: Option[] = partitions.map(p => ({
+        label: p._formatName,
+        value: p._name,
+      }));
+      setPartitionOptions(partitionOptions);
+    }
+  }, [collectionValue]);
+
+  useEffect(() => {
+    fetchPartition();
+  }, [fetchPartition]);
+
   const BackIcon = icons.back;
 
   // modal actions part, buttons label text or component
@@ -134,40 +142,71 @@ const InsertContainer: FC<InsertContentProps> = ({
     };
   }, [insertStatus]);
 
+  // props children component needed:
+  const collectionOptions: Option[] = useMemo(
+    () =>
+      defaultSelectedCollection === ''
+        ? collections.map(c => ({
+            label: c._name,
+            value: c._name,
+          }))
+        : [
+            {
+              label: defaultSelectedCollection,
+              value: defaultSelectedCollection,
+            },
+          ],
+    [collections, defaultSelectedCollection]
+  );
+
+  const schemaOptions: Option[] = useMemo(() => {
+    const list =
+      schema.length > 0
+        ? schema
+        : collections.find(c => c._name === collectionValue)?._fields;
+    return (list || []).map(s => ({
+      label: s._fieldName,
+      value: s._fieldId,
+    }));
+  }, [schema, collectionValue, collections]);
+
+  const [partitionOptions, setPartitionOptions] = useState<Option[]>([]);
+
   const checkUploadFileValidation = (fieldNamesLength: number): boolean => {
     return schemaOptions.length === fieldNamesLength;
   };
 
-  const previewData = useMemo(() => {
-    // we only show top 4 results of uploaded csv data
-    const end = isContainFieldNames ? 5 : 4;
-    return csvData.slice(0, end);
-  }, [csvData, isContainFieldNames]);
-
-  const handleUploadedData = (csv: string) => {
+  const handleUploadedData = (csv: string, uploader: HTMLFormElement) => {
     const { data } = parse(csv);
     const uploadFieldNamesLength = (data as string[])[0].length;
     const validation = checkUploadFileValidation(uploadFieldNamesLength);
     if (!validation) {
       // open snackbar
       openSnackBar(insertTrans('uploadFieldNamesLenWarning'), 'error');
-      // reset filename
+      // reset uploader value and filename
       setFileName('');
+      uploader.value = null;
       return;
     }
     setCsvData(data);
   };
 
   const handleInsertData = async () => {
+    // combine table heads and data
+    const tableData = isContainFieldNames ? csvData.slice(1) : csvData;
+
+    const data = combineHeadsAndData(tableHeads, tableData);
+
     setInsertStauts(InsertStatusEnum.loading);
-    const res = await handleInsert();
+    const res = await handleInsert(collectionValue, partitionValue, data);
     const status = res ? InsertStatusEnum.success : InsertStatusEnum.error;
     setInsertStauts(status);
   };
 
   const handleCollectionChange = (name: string) => {
     setCollectionValue(name);
-    handleSelectedCollectionChange && handleSelectedCollectionChange(name);
+    // reset partition
+    setPartitionValue('');
   };
 
   const handleNext = () => {
@@ -222,6 +261,8 @@ const InsertContainer: FC<InsertContentProps> = ({
           <InsertPreview
             schemaOptions={schemaOptions}
             data={previewData}
+            tableHeads={tableHeads}
+            setTableHeads={setTableHeads}
             isContainFieldNames={isContainFieldNames}
             handleIsContainedChange={setIsContainFieldNames}
           />

client/src/components/insert/Import.tsx

@@ -120,6 +120,7 @@ const InsertImport: FC<InsertImportProps> = ({
         <div className="selectorWrapper">
           <CustomSelector
             options={collectionOptions}
+            disabled={collectionOptions.length === 0}
             wrapperClass="selector"
             labelClass="selectLabel"
             value={selectedCollection}
@@ -134,6 +135,7 @@ const InsertImport: FC<InsertImportProps> = ({
           <Divider classes={{ root: 'divider' }} />
           <CustomSelector
             options={partitionOptions}
+            disabled={partitionOptions.length === 0}
             wrapperClass="selector"
             labelClass="selectLabel"
             value={selectedPartition}

client/src/components/insert/Preview.tsx

@@ -1,4 +1,4 @@
-import { FC, useCallback, useEffect, useMemo, useState } from 'react';
+import { FC, useCallback, useMemo } from 'react';
 import { makeStyles, Theme, Typography } from '@material-ui/core';
 import { useTranslation } from 'react-i18next';
 import { InsertPreviewProps } from './Types';
@@ -84,18 +84,13 @@ const getTableData = (
   return transferCsvArrayToTableData(csvData);
 };
 
-const getDefaultHeads = (
-  data: any[],
-  isContainFieldNames: number
-): string[] => {
-  return isContainFieldNames ? data[0] : new Array(data[0].length).fill('');
-};
-
 const InsertPreview: FC<InsertPreviewProps> = ({
   schemaOptions,
   data,
   isContainFieldNames,
   handleIsContainedChange,
+  tableHeads,
+  setTableHeads,
 }) => {
   const classes = getStyles();
   const { t: insertTrans } = useTranslation('insert');
@@ -104,24 +99,15 @@ const InsertPreview: FC<InsertPreviewProps> = ({
   // table needed table structure, metadata from csv
   const tableData = getTableData(data, isContainFieldNames);
 
-  const [tableHeads, setTableHeads] = useState<string[]>(
-    getDefaultHeads(data, isContainFieldNames)
-  );
-
   const handleTableHeadChange = useCallback(
     (index: number, label: string) => {
       const newHeads = [...tableHeads];
       newHeads[index] = label;
       setTableHeads(newHeads);
     },
-    [tableHeads]
+    [tableHeads, setTableHeads]
   );
 
-  useEffect(() => {
-    const newHeads = getDefaultHeads(data, isContainFieldNames);
-    setTableHeads(newHeads);
-  }, [data, isContainFieldNames]);
-
   const editHeads = useMemo(
     () =>
       tableHeads.map((head: string, index: number) => ({

client/src/components/insert/Types.ts

@@ -1,20 +1,26 @@
 import { CollectionData } from '../../pages/collections/Types';
-import { PartitionData } from '../../pages/partitions/Types';
 import { FieldData } from '../../pages/schema/Types';
 import { Option } from '../customSelector/Types';
 
 export interface InsertContentProps {
-  collections: CollectionData[];
-  partitions: PartitionData[];
+  // optional on partition page since its collection is fixed
+  collections?: CollectionData[];
+  // required on partition page since user can't select collection to get schema
+  schema?: FieldData[];
+
   // insert default selected collection
+  // if default value is not '', collections not selectable
   defaultSelectedCollection: string;
-  // optional if collections not selectable
-  handleSelectedCollectionChange?: (name: string) => void;
 
   // insert default selected partition
+  // if default value is not '', partitions not selectable
   defaultSelectedPartition: string;
-  schema: FieldData[];
-  handleInsert: () => Promise<boolean>;
+
+  handleInsert: (
+    collectionName: string,
+    partitionName: string,
+    fieldData: any[]
+  ) => Promise<boolean>;
 }
 
 export enum InsertStepperEnum {
@@ -44,7 +50,7 @@ export interface InsertImportProps {
   handleCollectionChange?: (collectionName: string) => void;
   handlePartitionChange: (partitionName: string) => void;
   // handle uploaded data
-  handleUploadedData: (data: string) => void;
+  handleUploadedData: (data: string, uploader: HTMLFormElement) => void;
   fileName: string;
   setFileName: (fileName: string) => void;
 }
@@ -53,6 +59,9 @@ export interface InsertPreviewProps {
   schemaOptions: Option[];
   data: any[];
 
+  tableHeads: string[];
+  setTableHeads: (heads: string[]) => void;
+
   isContainFieldNames: number;
   handleIsContainedChange: (isContained: number) => void;
 }

client/src/components/uploader/Types.ts

@@ -7,7 +7,9 @@ export interface UploaderProps {
   // snackbar warning when uploaded file size is over limit
   overSizeWarning?: string;
   setFileName: (fileName: string) => void;
-  handleUploadedData: (data: string) => void;
-  handleUploadFileChange?: (file: File) => void;
+  // handle uploader uploaded
+  handleUploadedData: (data: string, uploader: HTMLFormElement) => void;
+  // handle uploader onchange
+  handleUploadFileChange?: (file: File, uploader: HTMLFormElement) => void;
   handleUploadError?: () => void;
 }

client/src/components/uploader/Uploader.tsx

@@ -31,7 +31,7 @@ const Uploader: FC<UploaderProps> = ({
     reader.onload = async e => {
       const data = reader.result;
       if (data) {
-        handleUploadedData(data as string);
+        handleUploadedData(data as string, inputRef.current!);
       }
     };
     // handle upload error
@@ -55,7 +55,7 @@ const Uploader: FC<UploaderProps> = ({
       }
 
       setFileName(file.name || 'file');
-      handleUploadFileChange && handleUploadFileChange(file);
+      handleUploadFileChange && handleUploadFileChange(file, inputRef.current!);
       reader.readAsText(file, 'utf8');
     };
     uploader.click();

client/src/http/Collection.ts

@@ -1,5 +1,9 @@
 import { ChildrenStatusType, StatusEnum } from '../components/status/Types';
-import { CollectionView, DataType } from '../pages/collections/Types';
+import {
+  CollectionView,
+  DataType,
+  InsertDataParam,
+} from '../pages/collections/Types';
 import { IndexState, ShowCollectionsType } from '../types/Milvus';
 import { formatNumber } from '../utils/Common';
 import BaseModel from './BaseModel';
@@ -41,6 +45,13 @@ export class CollectionHttp extends BaseModel implements CollectionView {
     return super.findAll({ path: this.COLLECTIONS_URL, params: data || {} });
   }
 
+  static getCollection(name: string) {
+    return super.findAll({
+      path: `${this.COLLECTIONS_URL}/${name}`,
+      params: {},
+    });
+  }
+
   static createCollection(data: any) {
     return super.create({ path: this.COLLECTIONS_URL, data });
   }
@@ -72,6 +83,13 @@ export class CollectionHttp extends BaseModel implements CollectionView {
     return super.search({ path: this.COLLECTIONS_STATISTICS_URL, params: {} });
   }
 
+  static insertData(collectionName: string, param: InsertDataParam) {
+    return super.create({
+      path: `${this.COLLECTIONS_URL}/${collectionName}/insert`,
+      data: param,
+    });
+  }
+
   get _autoId() {
     return this.autoID;
   }

client/src/http/Milvus.ts

@@ -3,6 +3,7 @@ import BaseModel from './BaseModel';
 export class MilvusHttp extends BaseModel {
   static CONNECT_URL = '/milvus/connect';
   static CHECK_URL = '/milvus/check';
+  static FLUSH_URL = '/milvus/flush';
 
   constructor(props: {}) {
     super(props);
@@ -16,4 +17,13 @@ export class MilvusHttp extends BaseModel {
   static check(address: string) {
     return super.search({ path: this.CHECK_URL, params: { address } });
   }
+
+  static flush(collectionName: string) {
+    return super.update({
+      path: this.FLUSH_URL,
+      data: {
+        collection_names: [collectionName],
+      },
+    });
+  }
 }

client/src/pages/collections/Collections.tsx

@@ -4,7 +4,12 @@ import { useNavigationHook } from '../../hooks/Navigation';
 import { ALL_ROUTER_TYPES } from '../../router/Types';
 import MilvusGrid from '../../components/grid/Grid';
 import CustomToolBar from '../../components/grid/ToolBar';
-import { CollectionCreateParam, CollectionView, DataTypeEnum } from './Types';
+import {
+  CollectionCreateParam,
+  CollectionView,
+  DataTypeEnum,
+  InsertDataParam,
+} from './Types';
 import { ColDefinitionsType, ToolBarConfig } from '../../components/grid/Types';
 import { usePaginationHook } from '../../hooks/Pagination';
 import icons from '../../components/icons/Icons';
@@ -26,9 +31,7 @@ import {
 import Highlighter from 'react-highlight-words';
 import { parseLocationSearch } from '../../utils/Format';
 import InsertContainer from '../../components/insert/Container';
-import { PartitionData } from '../partitions/Types';
-import { FieldData } from '../schema/Types';
-import { PartitionHttp } from '../../http/Partition';
+import { MilvusHttp } from '../../http/Milvus';
 
 const useStyles = makeStyles((theme: Theme) => ({
   emptyWrapper: {
@@ -91,14 +94,6 @@ const Collections = () => {
   const ReleaseIcon = icons.release;
   const InfoIcon = icons.info;
 
-  /**
-   * insert needed data:
-   * 1. partitions: according to selected collection, always selectable
-   * 2. schema: according to selected collection, used as editable heads options
-   */
-  const [insertPartitions, setInsertPartitions] = useState<PartitionData[]>([]);
-  const [insertSchema, setInsertSchema] = useState<FieldData[]>([]);
-
   const fetchData = useCallback(async () => {
     try {
       const res = await CollectionHttp.getCollections();
@@ -144,27 +139,23 @@ const Collections = () => {
     fetchData();
   }, [fetchData]);
 
-  const handleInsert = useCallback(async (): Promise<boolean> => {
-    return new Promise((resolve, reject) => {});
-  }, []);
-
-  const handleInsertCollectionChange = useCallback(
-    async (name: string) => {
-      const selectCollection = collections.find(c => c._name === name);
-
-      console.log('select collection', selectCollection);
-      if (selectCollection) {
-        const partitions = await PartitionHttp.getPartitions(name);
-        console.log('----- partitions', partitions);
-        setInsertPartitions(partitions);
-
-        const schema = selectCollection._fields || [];
-        console.log('----- schema', schema);
-        setInsertSchema(schema);
-      }
-    },
-    [collections]
-  );
+  const handleInsert = async (
+    collectionName: string,
+    partitionName: string,
+    fieldData: any[]
+  ): Promise<boolean> => {
+    const param: InsertDataParam = {
+      partition_names: [partitionName],
+      fields_data: fieldData,
+    };
+    try {
+      await CollectionHttp.insertData(collectionName, param);
+      await MilvusHttp.flush(collectionName);
+      return true;
+    } catch (err) {
+      return false;
+    }
+  };
 
   const handleCreateCollection = async (param: CollectionCreateParam) => {
     const data: CollectionCreateParam = JSON.parse(JSON.stringify(param));
@@ -273,11 +264,8 @@ const Collections = () => {
                 ? selectedCollections[0]._name
                 : ''
             }
-            handleSelectedCollectionChange={handleInsertCollectionChange}
-            partitions={insertPartitions}
             // user can't select partition on collection page, so default value is ''
             defaultSelectedPartition={''}
-            schema={insertSchema}
             handleInsert={handleInsert}
           />
         );

client/src/pages/collections/Types.ts

@@ -77,3 +77,9 @@ export interface CreateFieldsProps {
   autoID: boolean;
   setAutoID: (value: boolean) => void;
 }
+
+export interface InsertDataParam {
+  partition_names: string[];
+  // e.g. [{vector: [1,2,3], age: 10}]
+  fields_data: any[];
+}

client/src/pages/partitions/Partitions.tsx

@@ -21,6 +21,9 @@ import { ManageRequestMethods } from '../../types/Common';
 import DeleteTemplate from '../../components/customDialog/DeleteDialogTemplate';
 import Highlighter from 'react-highlight-words';
 import { parseLocationSearch } from '../../utils/Format';
+import { useInsertDialogHook } from '../../hooks/Dialog';
+import InsertContainer from '../../components/insert/Container';
+import { CollectionHttp } from '../../http/Collection';
 
 const useStyles = makeStyles((theme: Theme) => ({
   wrapper: {
@@ -49,6 +52,8 @@ const Partitions: FC<{
   const { t: btnTrans } = useTranslation('btn');
   const { t: dialogTrans } = useTranslation('dialog');
   const InfoIcon = icons.info;
+
+  const { handleInsertDialog } = useInsertDialogHook();
   // const LoadIcon = icons.load;
   // const ReleaseIcon = icons.release;
 
@@ -102,6 +107,11 @@ const Partitions: FC<{
     [classes.highlight]
   );
 
+  const fetchCollectionDetail = async (name: string) => {
+    const res = await CollectionHttp.getCollection(name);
+    return res;
+  };
+
   useEffect(() => {
     fetchPartitions(collectionName);
   }, [collectionName, fetchPartitions]);
@@ -172,6 +182,10 @@ const Partitions: FC<{
     }, 300);
   };
 
+  const handleInsert = useCallback(async (): Promise<boolean> => {
+    return new Promise((resolve, reject) => {});
+  }, []);
+
   const toolbarConfigs: ToolBarConfig[] = [
     {
       label: t('create'),
@@ -191,6 +205,35 @@ const Partitions: FC<{
       },
       icon: 'add',
     },
+    {
+      label: btnTrans('insert'),
+      onClick: async () => {
+        const collection = await fetchCollectionDetail(collectionName);
+        // const schema = collection.schema.fields.map(f => new )
+        console.log('----- collections', collection);
+
+        handleInsertDialog(
+          <InsertContainer
+            collections={[]}
+            schema={[]}
+            defaultSelectedCollection={collectionName}
+            defaultSelectedPartition={
+              selectedPartitions.length === 1
+                ? selectedPartitions[0]._formatName
+                : ''
+            }
+            handleInsert={handleInsert}
+          />
+        );
+      },
+      /**
+       * insert validation:
+       * 1. At least 1 available collection
+       * 2. selected collections quantity shouldn't over 1
+       */
+      disabled: () => partitions.length === 0 || selectedPartitions.length > 1,
+      btnVariant: 'outlined',
+    },
     {
       type: 'iconBtn',
       onClick: () => {

client/src/utils/Insert.ts

@@ -11,3 +11,35 @@ export const transferCsvArrayToTableData = (data: any[][]) => {
     []
   );
 };
+
+const replaceKeysByIndex = (obj: any, newKeys: string[]) => {
+  const keyValues = Object.keys(obj).map(key => {
+    const newKey = newKeys[Number(key)] || key;
+    return { [newKey]: parseValue(obj[key]) };
+  });
+  return Object.assign({}, ...keyValues);
+};
+
+const parseValue = (value: string) => {
+  try {
+    return JSON.parse(value);
+  } catch (err) {
+    return value;
+  }
+};
+
+/**
+ *
+ * @param heads table heads, e.g. ['field1', 'field2', 'field3']
+ * @param data table data, e.g. [[23, [2,3,34,4,5,56], [1,1,1,1,1,1,1,1,1,1,1]]]
+ * @returns key value pair object array, with user selected heads or csv heads
+ */
+export const combineHeadsAndData = (heads: string[], data: any[]) => {
+  // use index as key, flatten two-dimensional array
+  // filter useless row
+  const flatTableData = data
+    .filter(d => d.some((item: string) => item !== ''))
+    .reduce((result, arr) => [...result, { ...arr }], []);
+  // replace flatTableData key with real head rely on index
+  return flatTableData.map((d: any) => replaceKeysByIndex(d, heads));
+};