Merge remote-tracking branch 'upstream/main' into feature/vector-search

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -5,27 +5,16 @@ labels: defect
 
 ---
 
-**Milvus-insight version:**
-
-**Milvus version:**
-
-**Browser version:**
+**Describe the bug:**
 
-**Browser OS version:**
 
-**Describe the bug:**
 
 **Steps to reproduce:**
 1.
 2.
 3.
 
-**Expected behavior:**
-
-**Screenshots (if relevant):**
-
-**Errors in browser console (if relevant):**
+**Milvus-insight version:**
 
-**Provide logs and/or server output (if relevant):**
 
-**Any additional context:**
+**Milvus version:**

.gitignore

@@ -32,6 +32,7 @@ server/dist
 server/build
 server/coverage
 server/documentation
+server/vectors.csv
 
 
 # package.lock.json

README.md

@@ -38,6 +38,8 @@ docker run -p 8000:3000 -e HOST_URL=http://127.0.0.1:8000 -e MILVUS_URL=127.0.0.
 
 Once you start the docker, open the browser, type `http://127.0.0.1:8000`, you can view the milvus insight.
 
+***note*** We plan to release milvus insight once a feature is done. Also, if you want to try the nightly build, please pull the docker image with the `dev` tag.
+
 ## ✨ Building and Running Milvus insight, and/or Contributing Code
 
 You might want to build Milvus-insight locally to contribute some code, test out the latest features, or try

client/src/components/insert/Import.tsx

@@ -160,7 +160,7 @@ const InsertImport: FC<InsertImportProps> = ({
             accept=".csv"
             setFileName={setFileName}
             handleUploadedData={handleUploadedData}
-            maxSize={parseByte('5m')}
+            maxSize={parseByte('150m')}
             overSizeWarning={insertTrans('overSizeWarning')}
           />
           <Typography className="text">

client/src/context/Auth.tsx

@@ -11,12 +11,14 @@ export const authContext = createContext<AuthContextType>({
 
 const { Provider } = authContext;
 export const AuthProvider = (props: { children: React.ReactNode }) => {
+  // get milvus address from local storage
   const [address, setAddress] = useState<string>(
     window.localStorage.getItem(MILVUS_ADDRESS) || ''
   );
   const isAuth = useMemo(() => !!address, [address]);
 
   useEffect(() => {
+    // check if the milvus is still available
     const check = async () => {
       const milvusAddress = window.localStorage.getItem(MILVUS_ADDRESS) || '';
       if (!milvusAddress) {
@@ -36,6 +38,12 @@ export const AuthProvider = (props: { children: React.ReactNode }) => {
     check();
   }, [setAddress]);
 
+  useEffect(() => {
+    document.title = address
+      ? `${address} - Milvus Insight`
+      : 'Milvus Insight';
+  }, [address]);
+
   return (
     <Provider value={{ isAuth, address, setAddress }}>
       {props.children}

client/src/http/Index.ts

@@ -6,6 +6,8 @@ import {
 } from '../pages/schema/Types';
 import { ManageRequestMethods } from '../types/Common';
 import { IndexState } from '../types/Milvus';
+import { findKeyValue } from '../utils/Common';
+import { getKeyValueListFromJsonString } from '../utils/Format';
 import BaseModel from './BaseModel';
 
 export class IndexHttp extends BaseModel implements IndexView {
@@ -62,7 +64,13 @@ export class IndexHttp extends BaseModel implements IndexView {
   }
 
   get _indexParameterPairs() {
-    return this.params.filter(p => p.key !== 'index_type');
+    const metricType = this.params.filter(v => v.key === 'metric_type');
+    // parms is json string, so we need parse it to key value array
+    const params = findKeyValue(this.params, 'params');
+    if (params) {
+      return [...metricType, ...getKeyValueListFromJsonString(params)];
+    }
+    return metricType;
   }
 
   get _fieldName() {

client/src/pages/connect/Connect.tsx

@@ -13,6 +13,7 @@ import { authContext } from '../../context/Auth';
 import { MilvusHttp } from '../../http/Milvus';
 import { rootContext } from '../../context/Root';
 import { MILVUS_ADDRESS } from '../../consts/Localstorage';
+import { formatAddress } from '../../utils/Format';
 
 const useStyles = makeStyles((theme: Theme) => ({
   wrapper: {
@@ -73,10 +74,11 @@ const Connect = () => {
   };
 
   const handleConnect = async () => {
-    await MilvusHttp.connect(form.address);
+    const address = formatAddress(form.address);
+    await MilvusHttp.connect(address);
     openSnackBar(successTrans('connect'));
-    setAddress(form.address);
-    window.localStorage.setItem(MILVUS_ADDRESS, form.address);
+    setAddress(address);
+    window.localStorage.setItem(MILVUS_ADDRESS, address);
     history.push('/');
   };
 

client/src/pages/schema/Create.tsx

@@ -57,6 +57,14 @@ const CreateIndex = (props: {
     [indexSetting.index_type, fieldType]
   );
 
+  const indexParams = useMemo(() => {
+    const params: { [x: string]: string } = {};
+    indexCreateParams.forEach(v => {
+      params[v] = indexSetting[v];
+    });
+    return params;
+  }, [indexCreateParams, indexSetting]);
+
   const indexOptions = useMemo(() => {
     const type =
       fieldType === 'BinaryVector'
@@ -77,6 +85,7 @@ const CreateIndex = (props: {
   const { validation, checkIsValid, disabled, setDisabled, resetValidation } =
     useFormValidation(checkedForm);
 
+  // reset index params
   useEffect(() => {
     setDisabled(true);
     setIndexSetting(v => ({
@@ -123,10 +132,10 @@ const CreateIndex = (props: {
         key: 'metric_type',
         value: metric_type,
       },
-      ...indexCreateParams.map(p => ({
-        key: p,
-        value: indexSetting[p],
-      })),
+      {
+        key: 'params',
+        value: JSON.stringify(indexParams),
+      },
     ];
 
     handleCreate(params);

client/src/pages/schema/Schema.tsx

@@ -99,6 +99,7 @@ const Schema: FC<{
 
       try {
         const list = await fetchSchemaListWithIndex(collectionName);
+        console.log(list);
         const fields: FieldView[] = list.map(f =>
           Object.assign(f, {
             _fieldNameElement: (

client/src/utils/Common.ts

@@ -33,3 +33,14 @@ export const formatNumber = (number: number): string => {
 export const throwErrorForDev = (text: string) => {
   throw new Error(text);
 };
+
+/**
+ *
+ * @param obj key value pair Array
+ * @param key the target you want to find.
+ * @returns undefined | string
+ */
+export const findKeyValue = (
+  obj: { key: string; value: string }[],
+  key: string
+) => obj.find(v => v.key === key)?.value;

client/src/utils/Format.ts

@@ -74,11 +74,11 @@ export const getEnumKeyByValue = (enumObj: any, enumValue: any) => {
   return '--';
 };
 
-export const getKeyValueListFromJSON = (
-  paramJSON: string
+export const getKeyValueListFromJsonString = (
+  json: string
 ): { key: string; value: string }[] => {
   try {
-    const obj = JSON.parse(paramJSON);
+    const obj = JSON.parse(json);
 
     const pairs: { key: string; value: string }[] = Object.entries(obj).map(
       ([key, value]) => ({
@@ -114,3 +114,6 @@ export const getCreateFieldType = (config: Field): CreateFieldType => {
 
   return 'number';
 };
+
+// Trim the address
+export const formatAddress = (address: string): string => address.trim();

server/generate-csv.ts

@@ -0,0 +1,33 @@
+import { createObjectCsvWriter as createCsvWriter } from 'csv-writer';
+
+// use to test vector insert
+const csvWriter = createCsvWriter({
+  path: './vectors.csv',
+  header: [
+    { id: 'vector', title: 'vector' },
+    { id: 'age', title: 'age' },
+  ],
+});
+
+const records = [];
+
+const generateVector = (dimension) => {
+  let index = 0;
+  const vectors = [];
+  while (index < dimension) {
+    vectors.push(1 + Math.random());
+    index++;
+  }
+  return JSON.stringify(vectors);
+};
+
+while (records.length < 50000) {
+  const value = generateVector(128);
+  records.push({ vector: value, age: 10 });
+}
+
+csvWriter
+  .writeRecords(records) // returns a promise
+  .then(() => {
+    console.log('...Done');
+  });

server/package.json

@@ -30,8 +30,9 @@
     "@nestjs/swagger": "^4.8.0",
     "@types/passport-jwt": "^3.0.5",
     "@types/passport-local": "^1.0.33",
-    "@zilliz/milvus2-sdk-node": "^1.0.2",
+    "@zilliz/milvus2-sdk-node": "^1.0.3",
     "body-parser": "^1.19.0",
+    "cache-manager": "^3.4.4",
     "class-transformer": "^0.4.0",
     "class-validator": "^0.13.1",
     "passport": "^0.4.1",
@@ -46,12 +47,14 @@
     "@nestjs/cli": "^7.6.0",
     "@nestjs/schematics": "^7.3.0",
     "@nestjs/testing": "^7.6.15",
+    "@types/cache-manager": "^3.4.2",
     "@types/express": "^4.17.11",
     "@types/jest": "^26.0.22",
     "@types/node": "^14.14.36",
     "@types/supertest": "^2.0.10",
     "@typescript-eslint/eslint-plugin": "^4.19.0",
     "@typescript-eslint/parser": "^4.19.0",
+    "csv-writer": "^1.6.0",
     "eslint": "^7.22.0",
     "eslint-config-prettier": "^8.1.0",
     "eslint-plugin-prettier": "^3.3.1",
@@ -81,4 +84,4 @@
     "coverageDirectory": "../coverage",
     "testEnvironment": "node"
   }
-}
+}

server/src/cache/config.ts

@@ -0,0 +1,5 @@
+export const ttl = 10; //seconds
+export const cacheKeys = {
+  LOADEDCOLLECTIONS: 'LOADEDCOLLECTIONS',
+  ALLCOLLECTIONS: 'ALLCOLLECTIONS',
+};

server/src/collections/collections.controller.ts

@@ -9,7 +9,12 @@ import {
   Query,
   UsePipes,
   ValidationPipe,
+  CACHE_MANAGER,
+  Inject,
+  UseInterceptors,
+  CacheInterceptor,
 } from '@nestjs/common';
+import { Cache } from 'cache-manager';
 import { ApiTags } from '@nestjs/swagger';
 import { CollectionsService } from './collections.service';
 import {
@@ -18,20 +23,38 @@ import {
   ShowCollections,
   VectorSearch,
 } from './dto';
+import { cacheKeys } from '../cache/config';
 
+//Including 2 kind of cache check getCollections and getStatistics for detail
 @ApiTags('collections')
 @Controller('collections')
 export class CollectionsController {
-  constructor(private collectionsService: CollectionsService) {}
+  constructor(private collectionsService: CollectionsService, @Inject(CACHE_MANAGER) private cacheManager: Cache) { }
 
+  // manually control cache if logic is complicated
   @Get()
   async getCollections(@Query() data?: ShowCollections) {
-    return Number(data.type) === 1
-      ? await this.collectionsService.getLoadedColletions()
-      : await this.collectionsService.getAllCollections();
+    if (Number(data.type) === 1) {
+      let loadedCollections = await this.cacheManager.get(cacheKeys.LOADEDCOLLECTIONS);
+      if (loadedCollections) {
+        return loadedCollections;
+      }
+      loadedCollections = await this.collectionsService.getLoadedColletions();
+      await this.cacheManager.set(cacheKeys.LOADEDCOLLECTIONS, loadedCollections);
+      return loadedCollections;
+    }
+    let allCollections = await this.cacheManager.get(cacheKeys.ALLCOLLECTIONS);
+    if (allCollections) {
+      return allCollections;
+    }
+    allCollections = await this.collectionsService.getAllCollections();
+    await this.cacheManager.set(cacheKeys.ALLCOLLECTIONS, allCollections);
+    return allCollections;
   }
 
+  // use interceptor to control cache automatically
   @Get('statistics')
+  @UseInterceptors(CacheInterceptor)
   async getStatistics() {
     return await this.collectionsService.getStatistics();
   }
@@ -39,12 +62,14 @@ export class CollectionsController {
   @Post()
   @UsePipes(new ValidationPipe())
   async createCollection(@Body() data: CreateCollection) {
+    await this.cacheManager.del(cacheKeys.ALLCOLLECTIONS);
     return await this.collectionsService.createCollection(data);
   }
 
   @Delete(':name')
   // todo: need check some special symbols
   async deleteCollection(@Param('name') name: string) {
+    await this.cacheManager.del(cacheKeys.ALLCOLLECTIONS);
     return await this.collectionsService.dropCollection({
       collection_name: name,
     });
@@ -71,6 +96,7 @@ export class CollectionsController {
 
   @Put(':name/load')
   async loadCollection(@Param('name') name: string) {
+    await this.cacheManager.del(cacheKeys.LOADEDCOLLECTIONS);
     return await this.collectionsService.loadCollection({
       collection_name: name,
     });
@@ -78,6 +104,7 @@ export class CollectionsController {
 
   @Put(':name/release')
   async releaseCollection(@Param('name') name: string) {
+    await this.cacheManager.del(cacheKeys.LOADEDCOLLECTIONS);
     return await this.collectionsService.releaseCollection({
       collection_name: name,
     });
@@ -85,6 +112,7 @@ export class CollectionsController {
 
   @Post(':name/insert')
   async insertData(@Param('name') name: string, @Body() data: InsertData) {
+    await this.cacheManager.del(cacheKeys.ALLCOLLECTIONS);
     return await this.collectionsService.insert({
       collection_name: name,
       ...data,

server/src/collections/collections.module.ts

@@ -1,11 +1,17 @@
-import { Module } from '@nestjs/common';
+import { Module, CacheModule } from '@nestjs/common';
 import { CollectionsService } from './collections.service';
 import { CollectionsController } from './collections.controller';
 import { MilvusModule } from '../milvus/milvus.module';
+import { ttl } from '../cache/config';
 
 @Module({
-  imports: [MilvusModule],
+  imports: [
+    MilvusModule,
+    CacheModule.register({
+      ttl, // seconds
+    }),
+  ],
   providers: [CollectionsService],
   controllers: [CollectionsController],
 })
-export class CollectionsModule {}
+export class CollectionsModule { }

server/src/main.ts

@@ -18,7 +18,7 @@ async function bootstrap() {
     .build();
   const document = SwaggerModule.createDocument(app, config);
   SwaggerModule.setup('api', app, document);
-  app.use(json({ limit: '50mb' }));
+  app.use(json({ limit: '150mb' }));
 
   await app.listen(port);
   Logger.log(`Milvus insight API server is running on port ${port}`);

server/yarn.lock

@@ -848,6 +848,11 @@
     "@types/connect" "*"
     "@types/node" "*"
 
+"@types/cache-manager@^3.4.2":
+  version "3.4.2"
+  resolved "https://registry.yarnpkg.com/@types/cache-manager/-/cache-manager-3.4.2.tgz#d57e7e5e6374d1037bdce753a05c9703e4483401"
+  integrity sha512-1IwA74t5ID4KWo0Kndal16MhiPSZgMe1fGc+MLT6j5r+Ab7jku36PFTl4PP6MiWw0BJscM9QpZEo00qixNQoRg==
+
 "@types/connect@*":
   version "3.4.34"
   resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.34.tgz#170a40223a6d666006d93ca128af2beb1d9b1901"
@@ -1292,10 +1297,10 @@
   resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d"
   integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==
 
-"@zilliz/milvus2-sdk-node@^1.0.2":
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/@zilliz/milvus2-sdk-node/-/milvus2-sdk-node-1.0.2.tgz#68aecdaa9d2f27058d9af8322e3fc7acaf31da1e"
-  integrity sha512-Uwz0OZqvu8hcaoUU5xXKkJpgx2xwDttzCJihqttB4iTMrn7HERZeFJZLYgeL3BbxdEgKF0jbxGlstZ2kUWx9fQ==
+"@zilliz/milvus2-sdk-node@^1.0.3":
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/@zilliz/milvus2-sdk-node/-/milvus2-sdk-node-1.0.3.tgz#909f1b44f5a7dedcfc3d075180f0273e8ace90f1"
+  integrity sha512-jAc/l7siEyNcDJ/WJ5tvbD/9bPMKXNWXCXaeUgLWLTitSrmjRLquhHKkf66OUU+rbEiDQw0l408jQyyY3CIZwA==
   dependencies:
     "@grpc/grpc-js" "^1.2.12"
     "@grpc/proto-loader" "^0.6.0"
@@ -1483,6 +1488,11 @@ astral-regex@^2.0.0:
   resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31"
   integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==
 
+async@3.2.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720"
+  integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==
+
 asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@@ -1731,6 +1741,15 @@ cache-base@^1.0.1:
     union-value "^1.0.0"
     unset-value "^1.0.0"
 
+cache-manager@^3.4.4:
+  version "3.4.4"
+  resolved "https://registry.yarnpkg.com/cache-manager/-/cache-manager-3.4.4.tgz#c69814763d3f3031395ae0d3a9a9296a91602226"
+  integrity sha512-oayy7ukJqNlRUYNUfQBwGOLilL0X5q7GpuaF19Yqwo6qdx49OoTZKRIF5qbbr+Ru8mlTvOpvnMvVq6vw72pOPg==
+  dependencies:
+    async "3.2.0"
+    lodash "^4.17.21"
+    lru-cache "6.0.0"
+
 call-bind@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c"
@@ -2104,6 +2123,11 @@ cssstyle@^2.3.0:
   dependencies:
     cssom "~0.3.6"
 
+csv-writer@^1.6.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/csv-writer/-/csv-writer-1.6.0.tgz#d0cea44b6b4d7d3baa2ecc6f3f7209233514bcf9"
+  integrity sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==
+
 data-urls@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b"
@@ -4091,7 +4115,7 @@ long@^4.0.0:
   resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"
   integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==
 
-lru-cache@^6.0.0:
+lru-cache@6.0.0, lru-cache@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94"
   integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==