Merge pull request #92 from nameczz/main

Add load collection api and change milvus-node-sdk-dev to milvus2-node-sdk

client/src/http/Collection.ts

@@ -1,6 +1,6 @@
 import { ChildrenStatusType, StatusEnum } from '../components/status/Types';
 import { CollectionView } from '../pages/collections/Types';
-import { IndexState } from '../types/Milvus';
+import { IndexState, ShowCollectionsType } from '../types/Milvus';
 import BaseModel from './BaseModel';
 
 export class CollectionHttp extends BaseModel implements CollectionView {
@@ -10,6 +10,7 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   private rowCount!: string;
   private index_status!: string;
   private id!: string;
+  private isLoaded!: boolean;
 
   static COLLECTIONS_URL = '/collections';
   static COLLECTIONS_INDEX_STATUS_URL = '/collections/indexes/status';
@@ -22,8 +23,10 @@ export class CollectionHttp extends BaseModel implements CollectionView {
     Object.assign(this, props);
   }
 
-  static getCollections(): Promise<CollectionHttp[]> {
-    return super.findAll({ path: this.COLLECTIONS_URL, params: {} });
+  static getCollections(data?: {
+    type: ShowCollectionsType;
+  }): Promise<CollectionHttp[]> {
+    return super.findAll({ path: this.COLLECTIONS_URL, params: data || {} });
   }
 
   static createCollection(data: any) {
@@ -78,7 +81,7 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   }
 
   get _status() {
-    return StatusEnum.loaded;
+    return this.isLoaded === true ? StatusEnum.loaded : StatusEnum.unloaded;
   }
 
   get _indexState() {

client/src/pages/overview/Overview.tsx

@@ -7,6 +7,7 @@ import { StatusEnum } from '../../components/status/Types';
 import { useNavigationHook } from '../../hooks/Navigation';
 import { CollectionHttp } from '../../http/Collection';
 import { ALL_ROUTER_TYPES } from '../../router/Types';
+import { ShowCollectionsType } from '../../types/Milvus';
 import { formatNumber } from '../../utils/Common';
 import CollectionCard from './collectionCard/CollectionCard';
 import { CollectionData } from './collectionCard/Types';
@@ -39,10 +40,16 @@ const Overview = () => {
     totalData: 0,
   });
 
+  const [loadCollections, setLoadCollections] = useState<CollectionHttp[]>([]);
+
   useEffect(() => {
     const fetchData = async () => {
       const res = await CollectionHttp.getStatistics();
+      const loadCollections = await CollectionHttp.getCollections({
+        type: ShowCollectionsType.InMemory,
+      });
       setStatistics(res);
+      setLoadCollections(loadCollections);
     };
     fetchData();
   }, []);
@@ -52,7 +59,7 @@ const Overview = () => {
       data: [
         {
           label: overviewTrans('load'),
-          value: formatNumber(4337),
+          value: formatNumber(loadCollections.length),
           valueColor: '#07d197',
         },
         {
@@ -69,46 +76,17 @@ const Overview = () => {
         },
       ],
     };
-  }, [overviewTrans, statistics]);
+  }, [overviewTrans, statistics, loadCollections]);
 
-  const mockCollections: CollectionData[] = [
-    {
-      name: 'collection1',
-      id: 'c1',
-      status: StatusEnum.loaded,
-      rowCount: 2,
-    },
-    {
-      name: 'collection2',
-      id: 'c2',
-      status: StatusEnum.loaded,
-      rowCount: 2,
-    },
-    {
-      name: 'collection3',
-      id: 'c3',
-      status: StatusEnum.loaded,
-      rowCount: 2,
-    },
-    {
-      name: 'collection4',
-      id: 'c4',
+  const loadCollectionsData: CollectionData[] = useMemo(() => {
+    return loadCollections.map(v => ({
+      id: v._id,
+      name: v._name,
       status: StatusEnum.loaded,
-      rowCount: 2,
-    },
-    {
-      name: 'collection5',
-      id: 'c5',
-      status: StatusEnum.loaded,
-      rowCount: 2,
-    },
-    {
-      name: 'collection6',
-      id: 'c6',
-      status: StatusEnum.loaded,
-      rowCount: 2,
-    },
-  ];
+      rowCount: Number(v._rowCount),
+    }));
+  }, [loadCollections]);
+
   const CollectionIcon = icons.navCollection;
 
   return (
@@ -117,9 +95,9 @@ const Overview = () => {
       <Typography className={classes.collectionTitle}>
         {overviewTrans('load')}
       </Typography>
-      {mockCollections.length > 0 ? (
+      {loadCollectionsData.length > 0 ? (
         <div className={classes.cardsWrapper}>
-          {mockCollections.map(collection => (
+          {loadCollectionsData.map(collection => (
             <CollectionCard key={collection.id} data={collection} />
           ))}
         </div>

client/src/pages/overview/collectionCard/CollectionCard.tsx

@@ -56,7 +56,7 @@ const CollectionCard: FC<CollectionCardProps> = ({
   wrapperClass = '',
 }) => {
   const classes = useStyles();
-  const { name, status, id, rowCount } = data;
+  const { name, status, rowCount } = data;
   const RightArrowIcon = icons.rightArrow;
   const InfoIcon = icons.info;
   const ReleaseIcon = icons.release;
@@ -73,7 +73,7 @@ const CollectionCard: FC<CollectionCardProps> = ({
       <Link
         classes={{ root: classes.link }}
         underline="none"
-        href={`/collection/${id}`}
+        href={`/collections/${name}`}
       >
         {name}
         <RightArrowIcon classes={{ root: classes.icon }} />

client/src/types/Milvus.ts

@@ -5,3 +5,8 @@ export enum IndexState {
   Finished = 'Finished',
   Failed = 'Failed',
 }
+
+export enum ShowCollectionsType {
+  All = 0,
+  InMemory = 1,
+}

server/package.json

@@ -30,7 +30,7 @@
     "@nestjs/swagger": "^4.8.0",
     "@types/passport-jwt": "^3.0.5",
     "@types/passport-local": "^1.0.33",
-    "@zilliz/milvus-sdk-node-dev": "^0.1.4",
+    "@zilliz/milvus2-sdk-node": "^0.1.0",
     "class-transformer": "^0.4.0",
     "class-validator": "^0.13.1",
     "passport": "^0.4.1",

server/src/collections/collections.controller.ts

@@ -6,12 +6,13 @@ import {
   Param,
   Post,
   Put,
+  Query,
   UsePipes,
   ValidationPipe,
 } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import { CollectionsService } from './collections.service';
-import { CreateCollection } from './dto';
+import { CreateCollection, ShowCollections } from './dto';
 
 @ApiTags('collections')
 @Controller('collections')
@@ -19,8 +20,10 @@ export class CollectionsController {
   constructor(private collectionsService: CollectionsService) {}
 
   @Get()
-  async getCollections() {
-    return await this.collectionsService.showCollections();
+  async getCollections(@Query() data?: ShowCollections) {
+    return Number(data.type) === 1
+      ? await this.collectionsService.getLoadedColletions()
+      : await this.collectionsService.getAllCollections();
   }
 
   @Get('statistics')

server/src/collections/collections.service.ts

@@ -8,10 +8,14 @@ import {
   GetIndexStateReq,
   LoadCollectionReq,
   ReleaseLoadCollectionReq,
-} from '@zilliz/milvus-sdk-node-dev/dist/milvus/types';
+} from '@zilliz/milvus2-sdk-node/dist/milvus/types';
 import { throwErrorFromSDK } from '../utils/Error';
 import { findKeyValue } from '../utils/Helper';
 import { ROW_COUNT } from '../utils/Const';
+import {
+  ShowCollectionsReq,
+  ShowCollectionsType,
+} from '@zilliz/milvus2-sdk-node/dist/milvus/types/Collection';
 @Injectable()
 export class CollectionsService {
   constructor(private milvusService: MilvusService) {}
@@ -20,8 +24,8 @@ export class CollectionsService {
     return this.milvusService.milvusClientGetter;
   }
 
-  async getCollectionNames() {
-    const res = await this.milvusClient.showCollections();
+  async getCollectionNames(data?: ShowCollectionsReq) {
+    const res = await this.milvusClient.showCollections(data);
     throwErrorFromSDK(res.status);
     return res;
   }
@@ -78,9 +82,12 @@ export class CollectionsService {
    * Get all collections meta data
    * @returns {id:string, collection_name:string, schema:Field[], autoID:boolean, rowCount: string}
    */
-  async showCollections() {
+  async getAllCollections() {
     const data = [];
     const res = await this.getCollectionNames();
+    const loadedCollections = await this.getCollectionNames({
+      type: ShowCollectionsType.InMemory,
+    });
     if (res.collection_names.length > 0) {
       for (const name of res.collection_names) {
         const collectionInfo = await this.describeCollection({
@@ -96,6 +103,27 @@ export class CollectionsService {
           autoID: collectionInfo.schema.autoID,
           rowCount: findKeyValue(collectionStatistics.stats, ROW_COUNT),
           id: collectionInfo.collectionID,
+          isLoaded: loadedCollections.collection_names.includes(name),
+        });
+      }
+    }
+    return data;
+  }
+
+  async getLoadedColletions() {
+    const data = [];
+    const res = await this.getCollectionNames({
+      type: ShowCollectionsType.InMemory,
+    });
+    if (res.collection_names.length > 0) {
+      for (const [index, value] of res.collection_names.entries()) {
+        const collectionStatistics = await this.getCollectionStatistics({
+          collection_name: value,
+        });
+        data.push({
+          id: res.collection_ids[index],
+          collection_name: value,
+          rowCount: findKeyValue(collectionStatistics.stats, ROW_COUNT),
         });
       }
     }

server/src/collections/dto.ts

@@ -5,15 +5,18 @@ import {
   IsOptional,
   IsArray,
   ArrayNotEmpty,
+  IsEnum,
 } from 'class-validator';
-import { FieldType } from '@zilliz/milvus-sdk-node-dev/dist/milvus/types/Collection'; // todo: need improve like export types in root file.
-import { DataType } from '@zilliz/milvus-sdk-node-dev/dist/milvus/types/Common';
+import {
+  FieldType,
+  ShowCollectionsType,
+} from '@zilliz/milvus2-sdk-node/dist/milvus/types/Collection'; // todo: need improve like export types in root file.
+import { DataType } from '@zilliz/milvus2-sdk-node/dist/milvus/types/Common';
 import { ApiProperty } from '@nestjs/swagger';
 
-
 export class CreateCollection {
   @ApiProperty({
-    description: 'Milvus collection name'
+    description: 'Milvus collection name',
   })
   @IsString()
   @IsNotEmpty({
@@ -31,7 +34,7 @@ export class CreateCollection {
 
   @ApiProperty({
     description: 'Field data type',
-    enum: DataType
+    enum: DataType,
   })
   @IsArray()
   @ArrayNotEmpty()
@@ -40,3 +43,13 @@ export class CreateCollection {
   })
   readonly fields: FieldType[];
 }
+
+export class ShowCollections {
+  @ApiProperty({
+    description: 'Type allow all->0 inmemory->1',
+    enum: ShowCollectionsType,
+  })
+  @IsOptional()
+  @IsEnum(ShowCollectionsType, { message: 'Type allow all->0 inmemory->1' })
+  readonly type: ShowCollectionsType;
+}

server/src/main.ts

@@ -6,6 +6,7 @@ async function bootstrap() {
   const app = await NestFactory.create(AppModule, {
     cors: true,
   });
+  app.setGlobalPrefix('/api/v1');
 
   const config = new DocumentBuilder()
     .setTitle('Milvus insight')
@@ -14,7 +15,6 @@ async function bootstrap() {
   const document = SwaggerModule.createDocument(app, config);
   SwaggerModule.setup('api', app, document);
 
-  app.setGlobalPrefix('/api/v1');
   await app.listen(3000);
 }
 bootstrap();

server/src/milvus/milvus.service.ts

@@ -1,5 +1,5 @@
 import { Injectable } from '@nestjs/common';
-import { MilvusNode } from '@zilliz/milvus-sdk-node-dev';
+import { MilvusNode } from '@zilliz/milvus2-sdk-node';
 @Injectable()
 export class MilvusService {
   private milvusAddress: string;

server/src/partitions/partitions.service.ts

@@ -7,7 +7,7 @@ import {
   LoadPartitionsReq,
   ReleasePartitionsReq,
   ShowPartitionsReq,
-} from '@zilliz/milvus-sdk-node-dev/dist/milvus/types'; // todo: need improve like export types in root file.
+} from '@zilliz/milvus2-sdk-node/dist/milvus/types'; // todo: need improve like export types in root file.
 import { throwErrorFromSDK } from 'src/utils/Error';
 import { findKeyValue } from '../utils/Helper';
 import { ROW_COUNT } from '../utils/Const';

server/src/schema/schema.service.ts

@@ -5,7 +5,7 @@ import {
   DropIndexReq,
   GetIndexBuildProgressReq,
   GetIndexStateReq,
-} from '@zilliz/milvus-sdk-node-dev/dist/milvus/types';
+} from '@zilliz/milvus2-sdk-node/dist/milvus/types';
 import { throwErrorFromSDK } from 'src/utils/Error';
 import { MilvusService } from '../milvus/milvus.service';
 

server/src/utils/Error.ts

@@ -1,7 +1,7 @@
 import {
   ErrorCode,
   ResStatus,
-} from '@zilliz/milvus-sdk-node-dev/dist/milvus/types/Response';
+} from '@zilliz/milvus2-sdk-node/dist/milvus/types/Response';
 
 export const throwErrorFromSDK = (res: ResStatus) => {
   if (res.error_code !== ErrorCode.SUCCESS) {

server/src/utils/Helper.ts

@@ -1,4 +1,4 @@
-import { KeyValuePair } from '@zilliz/milvus-sdk-node-dev/dist/milvus/types/Common';
+import { KeyValuePair } from '@zilliz/milvus2-sdk-node/dist/milvus/types/Common';
 
 export const findKeyValue = (obj: KeyValuePair[], key: string) =>
   obj.find((v) => v.key === key)?.value;

server/yarn.lock

@@ -1287,10 +1287,10 @@
   resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d"
   integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==
 
-"@zilliz/milvus-sdk-node-dev@^0.1.4":
-  version "0.1.4"
-  resolved "https://registry.yarnpkg.com/@zilliz/milvus-sdk-node-dev/-/milvus-sdk-node-dev-0.1.4.tgz#858e83b0ba0a309cbdf0b87e313b3b41de0052a4"
-  integrity sha512-4REU6TxqjsIZ7a7mL1M19IPg3oBBfE9kJmC7XUmcFz1+F2fYWpl7Iw4R8g69trpzUNnaFIBgwjAnSb0oVrRLeQ==
+"@zilliz/milvus2-sdk-node@^0.1.0":
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/@zilliz/milvus2-sdk-node/-/milvus2-sdk-node-0.1.0.tgz#feab0f956a84f73e228d40ce4efc358e86c97e60"
+  integrity sha512-sehxvOndtqFknKdHYSaoks1kZo5q8t72mHqWiduIwsuIsCkpQMoOUyHOpWXvakWajrKrfksIdgn2oaUj5/JcBQ==
   dependencies:
     "@grpc/grpc-js" "^1.2.12"
     "@grpc/proto-loader" "^0.6.0"