Merge branch 'main' into uxd-624

server/src/app.module.ts

@@ -10,6 +10,8 @@ import { UsersModule } from './users/users.module';
 import { AuthModule } from './auth/auth.module';
 import { ServeStaticModule } from '@nestjs/serve-static';
 import { join } from 'path';
+import { PartitionsModule } from './partitions/partitions.module';
+import { SchemaModule } from './schema/schema.module';
 
 @Module({
   imports: [
@@ -21,6 +23,8 @@ import { join } from 'path';
     CollectionsModule,
     UsersModule,
     AuthModule,
+    PartitionsModule,
+    SchemaModule,
   ],
   controllers: [AppController],
   providers: [

server/src/collections/collections.service.ts

@@ -8,8 +8,10 @@ import {
   GetIndexStateReq,
   LoadCollectionReq,
   ReleaseLoadCollectionReq,
-} from '@zilliz/milvus-sdk-node-dev/dist/milvus/types'; // todo: need improve like export types in root file.
-import { throwErrorFromSDK } from 'src/utils/Error';
+} from '@zilliz/milvus-sdk-node-dev/dist/milvus/types';
+import { throwErrorFromSDK } from '../utils/Error';
+import { findKeyValue } from '../utils/Helper';
+import { ROW_COUNT } from '../utils/Const';
 @Injectable()
 export class CollectionsService {
   constructor(private milvusService: MilvusService) {}
@@ -25,13 +27,9 @@ export class CollectionsService {
   }
 
   async createCollection(data: CreateCollectionReq) {
-    try {
-      const res = await this.milvusClient.createCollection(data);
-      throwErrorFromSDK(res);
-      return res;
-    } catch (error) {
-      throw new Error(error);
-    }
+    const res = await this.milvusClient.createCollection(data);
+    throwErrorFromSDK(res);
+    return res;
   }
 
   async describeCollection(data: DescribeCollectionReq) {
@@ -89,12 +87,10 @@ export class CollectionsService {
         });
         data.push({
           collection_name: name,
-          // schema: collectionInfo.schema,
+          schema: collectionInfo.schema,
           description: collectionInfo.schema.description,
           autoID: collectionInfo.schema.autoID,
-          rowCount: collectionStatistics.stats.find(
-            (v) => v.key === 'row_count',
-          ).value,
+          rowCount: findKeyValue(collectionStatistics.stats, ROW_COUNT),
           // id: collectionInfo.collectionId
         });
       }

server/src/collections/dto.ts

@@ -7,18 +7,22 @@ import {
   ArrayNotEmpty,
 } from 'class-validator';
 import { FieldType } from '@zilliz/milvus-sdk-node-dev/dist/milvus/types/Collection'; // todo: need improve like export types in root file.
+import { ApiProperty } from '@nestjs/swagger';
 
 export class CreateCollection {
+  @ApiProperty()
   @IsString()
   @IsNotEmpty({
     message: 'collection_name is empty',
   })
   readonly collection_name: string;
 
+  @ApiProperty()
   @IsBoolean()
   @IsOptional()
   readonly autoID: boolean;
 
+  @ApiProperty()
   @IsArray()
   @ArrayNotEmpty()
   @IsNotEmpty({

server/src/milvus/dto.ts

@@ -1,6 +1,8 @@
+import { ApiProperty } from '@nestjs/swagger';
 import { IsNotEmpty, IsString } from 'class-validator';
 
 export class ConnectMilvus {
+  @ApiProperty()
   @IsString()
   @IsNotEmpty({
     message: 'address is empty',
@@ -9,6 +11,7 @@ export class ConnectMilvus {
 }
 
 export class CheckMilvus {
+  @ApiProperty()
   @IsString()
   @IsNotEmpty({
     message: 'address is empty',

server/src/milvus/milvus.service.ts

@@ -28,6 +28,7 @@ export class MilvusService {
       });
       return { address: this.milvusAddress };
     } catch (error) {
+      console.log(error);
       throw new Error('Connect milvus failed, check your milvus address.');
     }
   }

server/src/partitions/dto.ts

@@ -0,0 +1,58 @@
+import { ApiProperty } from '@nestjs/swagger';
+import {
+  IsNotEmpty,
+  IsString,
+  IsEnum,
+  IsArray,
+  ArrayNotEmpty,
+} from 'class-validator';
+
+export enum ManageType {
+  DELETE = 'delete',
+  CREATE = 'create',
+}
+export class GetPartitionsInfo {
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty({
+    message: 'collection_name is empty',
+  })
+  readonly collection_name: string;
+}
+
+export class ManagePartition {
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty({
+    message: 'collection_name is empty',
+  })
+  readonly collection_name: string;
+
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty({
+    message: 'partition_name is empty',
+  })
+  readonly partition_name: string;
+
+  @ApiProperty({ enum: ManageType })
+  @IsEnum(ManageType, { message: 'Type allow delete and create' })
+  readonly type: ManageType;
+}
+
+export class LoadPartitions {
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty({
+    message: 'collection_name is empty',
+  })
+  readonly collection_name: string;
+
+  @ApiProperty()
+  @IsArray()
+  @ArrayNotEmpty()
+  @IsNotEmpty({
+    message: 'partition_names is empty',
+  })
+  readonly partition_names: string[];
+}

server/src/partitions/partitions.controller.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { PartitionsController } from './partitions.controller';
+
+describe('PartitionsController', () => {
+  let controller: PartitionsController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [PartitionsController],
+    }).compile();
+
+    controller = module.get<PartitionsController>(PartitionsController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

server/src/partitions/partitions.controller.ts

@@ -0,0 +1,50 @@
+import {
+  Body,
+  Controller,
+  Get,
+  Post,
+  Put,
+  Query,
+  UsePipes,
+  ValidationPipe,
+} from '@nestjs/common';
+import {
+  GetPartitionsInfo,
+  LoadPartitions,
+  ManagePartition,
+  ManageType,
+} from './dto';
+import { PartitionsService } from './partitions.service';
+
+@Controller('partitions')
+export class PartitionsController {
+  constructor(private partitionsService: PartitionsService) {}
+
+  @Get()
+  @UsePipes(new ValidationPipe())
+  async getPartitions(@Query() query: GetPartitionsInfo) {
+    return await this.partitionsService.getPatitionsInfo(query);
+  }
+
+  @Post()
+  @UsePipes(new ValidationPipe())
+  async managePartition(@Body() body: ManagePartition) {
+    const { type, ...params } = body;
+
+    return type.toLocaleLowerCase() === ManageType.CREATE
+      ? await this.partitionsService.createParition(params)
+      : await this.partitionsService.deleteParition(params);
+  }
+
+  @Put('load')
+  @UsePipes(new ValidationPipe())
+  async loadPartition(@Body() body: LoadPartitions) {
+    return await this.partitionsService.loadPartitions(body);
+  }
+
+  @Put('release')
+  @UsePipes(new ValidationPipe())
+  async releasePartitions(@Body() body: LoadPartitions) {
+    return await this.partitionsService.loadPartitions(body);
+  }
+}

server/src/partitions/partitions.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { PartitionsService } from './partitions.service';
+import { PartitionsController } from './partitions.controller';
+import { MilvusModule } from 'src/milvus/milvus.module';
+
+@Module({
+  imports: [MilvusModule],
+  providers: [PartitionsService],
+  controllers: [PartitionsController],
+})
+export class PartitionsModule {}

server/src/partitions/partitions.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { PartitionsService } from './partitions.service';
+
+describe('PartitionsService', () => {
+  let service: PartitionsService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [PartitionsService],
+    }).compile();
+
+    service = module.get<PartitionsService>(PartitionsService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

server/src/partitions/partitions.service.ts

@@ -0,0 +1,77 @@
+import { Injectable } from '@nestjs/common';
+import { MilvusService } from '../milvus/milvus.service';
+import {
+  CreatePartitionReq,
+  DropPartitionReq,
+  GetPartitionStatisticsReq,
+  LoadPartitionsReq,
+  ReleasePartitionsReq,
+  ShowPartitionsReq,
+} from '@zilliz/milvus-sdk-node-dev/dist/milvus/types'; // todo: need improve like export types in root file.
+import { throwErrorFromSDK } from 'src/utils/Error';
+import { findKeyValue } from '../utils/Helper';
+import { ROW_COUNT } from '../utils/Const';
+
+@Injectable()
+export class PartitionsService {
+  constructor(private milvusService: MilvusService) {}
+
+  get milvusClient() {
+    return this.milvusService.milvusClientGetter;
+  }
+
+  async getPatitionsInfo(data: ShowPartitionsReq) {
+    const result = [];
+    const res = await this.getPartitions(data);
+    if (res.partition_names && res.partition_names.length) {
+      for (const [index, name] of res.partition_names.entries()) {
+        const statistics = await this.getPartitionStatistics({
+          ...data,
+          partition_name: name,
+        });
+        result.push({
+          name,
+          id: res.partitionIDs[index],
+          rowCount: findKeyValue(statistics.stats, ROW_COUNT),
+        });
+      }
+    }
+    return result;
+  }
+
+  async getPartitions(data: ShowPartitionsReq) {
+    const res = await this.milvusClient.showPartitions(data);
+    throwErrorFromSDK(res.status);
+    return res;
+  }
+
+  async createParition(data: CreatePartitionReq) {
+    const res = await this.milvusClient.createPartition(data);
+    throwErrorFromSDK(res);
+    return res;
+  }
+
+  async deleteParition(data: DropPartitionReq) {
+    const res = await this.milvusClient.dropPartition(data);
+    throwErrorFromSDK(res);
+    return res;
+  }
+
+  async getPartitionStatistics(data: GetPartitionStatisticsReq) {
+    const res = await this.milvusClient.getPartitionStatistics(data);
+    throwErrorFromSDK(res.status);
+    return res;
+  }
+
+  async loadPartitions(data: LoadPartitionsReq) {
+    const res = await this.milvusClient.loadPartitions(data);
+    throwErrorFromSDK(res);
+    return res;
+  }
+
+  async releasePartitions(data: ReleasePartitionsReq) {
+    const res = await this.milvusClient.releasePartitions(data);
+    throwErrorFromSDK(res);
+    return res;
+  }
+}

server/src/schema/dto.ts

@@ -0,0 +1,96 @@
+import { ApiProperty } from '@nestjs/swagger';
+import {
+  IsNotEmpty,
+  IsString,
+  IsArray,
+  IsEnum,
+  IsOptional,
+} from 'class-validator';
+
+class KeyValuePair {
+  @ApiProperty()
+  key: string;
+  @ApiProperty()
+  value: string;
+}
+
+export enum ManageType {
+  DELETE = 'delete',
+  CREATE = 'create',
+}
+
+export class ManageIndex {
+  @ApiProperty({ enum: ManageType })
+  @IsEnum(ManageType, { message: 'Type allow delete and create' })
+  readonly type: ManageType;
+
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty({
+    message: 'collection_name is empty',
+  })
+  readonly collection_name: string;
+
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty({
+    message: 'field_name is empty',
+  })
+  readonly field_name: string;
+
+  @ApiProperty({
+    type: [KeyValuePair],
+  })
+  @IsArray()
+  @IsOptional()
+  readonly extra_params?: KeyValuePair[];
+}
+
+export class DescribeIndex {
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty({
+    message: 'collection_name is empty',
+  })
+  readonly collection_name: string;
+
+  @ApiProperty()
+  @IsString()
+  @IsOptional()
+  readonly field_name?: string;
+}
+
+export class GetIndexState {
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty({
+    message: 'collection_name is empty',
+  })
+  readonly collection_name: string;
+
+  @ApiProperty()
+  @IsString()
+  @IsOptional()
+  readonly field_name?: string;
+}
+
+export class GetIndexProgress {
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty({
+    message: 'collection_name is empty',
+  })
+  readonly collection_name: string;
+
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty({
+    message: 'index_name is empty',
+  })
+  readonly index_name: string;
+
+  @ApiProperty()
+  @IsString()
+  @IsOptional()
+  readonly field_name?: string;
+}

server/src/schema/schema.controller.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { SchemaController } from './schema.controller';
+
+describe('SchemaController', () => {
+  let controller: SchemaController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [SchemaController],
+    }).compile();
+
+    controller = module.get<SchemaController>(SchemaController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

server/src/schema/schema.controller.ts

@@ -0,0 +1,53 @@
+import {
+  Body,
+  Controller,
+  Get,
+  Post,
+  Query,
+  UsePipes,
+  ValidationPipe,
+} from '@nestjs/common';
+import {
+  DescribeIndex,
+  GetIndexProgress,
+  ManageIndex,
+  ManageType,
+  GetIndexState,
+} from './dto';
+import { SchemaService } from './schema.service';
+
+@Controller('schema')
+export class SchemaController {
+  constructor(private schemaService: SchemaService) {}
+
+  @Post('index')
+  @UsePipes(new ValidationPipe())
+  async manageIndex(@Body() body: ManageIndex) {
+    const { type, collection_name, extra_params, field_name } = body;
+    return type === ManageType.CREATE
+      ? await this.schemaService.createIndex({
+          collection_name,
+          extra_params,
+          field_name,
+        })
+      : await this.schemaService.dropIndex({ collection_name, field_name });
+  }
+
+  @Get('index')
+  @UsePipes(new ValidationPipe())
+  async describeIndex(@Query() query: DescribeIndex) {
+    return await this.schemaService.describeIndex(query);
+  }
+
+  @Get('index/progress')
+  @UsePipes(new ValidationPipe())
+  async getIndexProgress(@Query() query: GetIndexProgress) {
+    return await this.schemaService.getIndexBuildProgress(query);
+  }
+
+  @Get('index/state')
+  @UsePipes(new ValidationPipe())
+  async getIndexState(@Query() query: GetIndexState) {
+    return await this.schemaService.getIndexState(query);
+  }
+}

server/src/schema/schema.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { MilvusModule } from '../milvus/milvus.module';
+import { SchemaController } from './schema.controller';
+import { SchemaService } from './schema.service';
+
+@Module({
+  imports: [MilvusModule],
+  controllers: [SchemaController],
+  providers: [SchemaService],
+})
+export class SchemaModule {}

server/src/schema/schema.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { SchemaService } from './schema.service';
+
+describe('SchemaService', () => {
+  let service: SchemaService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [SchemaService],
+    }).compile();
+
+    service = module.get<SchemaService>(SchemaService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

server/src/schema/schema.service.ts

@@ -0,0 +1,49 @@
+import { Injectable } from '@nestjs/common';
+import {
+  CreateIndexReq,
+  DescribeIndexReq,
+  DropIndexReq,
+  GetIndexBuildProgressReq,
+  GetIndexStateReq,
+} from '@zilliz/milvus-sdk-node-dev/dist/milvus/types';
+import { throwErrorFromSDK } from 'src/utils/Error';
+import { MilvusService } from '../milvus/milvus.service';
+
+@Injectable()
+export class SchemaService {
+  constructor(private milvusService: MilvusService) {}
+
+  get milvusClient() {
+    return this.milvusService.milvusClientGetter;
+  }
+
+  async createIndex(data: CreateIndexReq) {
+    const res = await this.milvusClient.createIndex(data);
+    throwErrorFromSDK(res);
+    return res;
+  }
+
+  async describeIndex(data: DescribeIndexReq) {
+    const res = await this.milvusClient.describeIndex(data);
+    throwErrorFromSDK(res.status);
+    return res;
+  }
+
+  async dropIndex(data: DropIndexReq) {
+    const res = await this.milvusClient.dropIndex(data);
+    throwErrorFromSDK(res);
+    return res;
+  }
+
+  async getIndexState(data: GetIndexStateReq) {
+    const res = await this.milvusClient.getIndexState(data);
+    throwErrorFromSDK(res.status);
+    return res;
+  }
+
+  async getIndexBuildProgress(data: GetIndexBuildProgressReq) {
+    const res = await this.milvusClient.getIndexBuildProgress(data);
+    throwErrorFromSDK(res.status);
+    return res;
+  }
+}

server/src/utils/Const.ts

@@ -0,0 +1 @@
+export const ROW_COUNT = 'row_count';

server/src/utils/Helper.ts

@@ -0,0 +1,4 @@
+import { KeyValuePair } from '@zilliz/milvus-sdk-node-dev/dist/milvus/types/Common';
+
+export const findKeyValue = (obj: KeyValuePair[], key: string) =>
+  obj.find((v) => v.key === key)?.value;