|
@@ -20,8 +20,8 @@ import {
|
|
|
CompactReq,
|
|
|
HasCollectionReq,
|
|
|
CountReq,
|
|
|
- FieldSchema,
|
|
|
GetLoadStateReq,
|
|
|
+ CollectionData,
|
|
|
} from '@zilliz/milvus2-sdk-node';
|
|
|
import { Parser } from '@json2csv/plainjs';
|
|
|
import {
|
|
@@ -30,11 +30,21 @@ import {
|
|
|
genRows,
|
|
|
ROW_COUNT,
|
|
|
convertFieldSchemaToFieldType,
|
|
|
+ LOADING_STATE,
|
|
|
} from '../utils';
|
|
|
import { QueryDto, ImportSampleDto, GetReplicasDto } from './dto';
|
|
|
-import { CollectionData } from '../types';
|
|
|
+import {
|
|
|
+ CollectionObject,
|
|
|
+ CollectionLazyObject,
|
|
|
+ FieldObject,
|
|
|
+ IndexObject,
|
|
|
+ DescribeCollectionRes,
|
|
|
+ CountObject,
|
|
|
+ StatisticsObject,
|
|
|
+} from '../types';
|
|
|
import { SchemaService } from '../schema/schema.service';
|
|
|
import { clientCache } from '../app';
|
|
|
+import { MIN_INT64 } from '../utils/Const';
|
|
|
|
|
|
export class CollectionsService {
|
|
|
private schemaService: SchemaService;
|
|
@@ -43,7 +53,7 @@ export class CollectionsService {
|
|
|
this.schemaService = new SchemaService();
|
|
|
}
|
|
|
|
|
|
- async getCollections(clientId: string, data?: ShowCollectionsReq) {
|
|
|
+ async showCollections(clientId: string, data?: ShowCollectionsReq) {
|
|
|
const { milvusClient } = clientCache.get(clientId);
|
|
|
const res = await milvusClient.showCollections(data);
|
|
|
throwErrorFromSDK(res.status);
|
|
@@ -59,8 +69,60 @@ export class CollectionsService {
|
|
|
|
|
|
async describeCollection(clientId: string, data: DescribeCollectionReq) {
|
|
|
const { milvusClient } = clientCache.get(clientId);
|
|
|
- const res = await milvusClient.describeCollection(data);
|
|
|
+ const res = (await milvusClient.describeCollection(
|
|
|
+ data
|
|
|
+ )) as DescribeCollectionRes;
|
|
|
+
|
|
|
+ // get index info for collections
|
|
|
+ const indexRes = await this.schemaService.describeIndex(clientId, {
|
|
|
+ collection_name: data.collection_name,
|
|
|
+ });
|
|
|
+
|
|
|
throwErrorFromSDK(res.status);
|
|
|
+
|
|
|
+ const vectorFields: FieldObject[] = [];
|
|
|
+ const scalarFields: FieldObject[] = [];
|
|
|
+
|
|
|
+ // append index info to each field
|
|
|
+ res.schema.fields.forEach((field: FieldObject) => {
|
|
|
+ // add index
|
|
|
+ field.index = indexRes.index_descriptions.find(
|
|
|
+ index => index.field_name === field.name
|
|
|
+ ) as IndexObject;
|
|
|
+ // add dimension
|
|
|
+ field.dimension =
|
|
|
+ Number(field.type_params.find(item => item.key === 'dim')?.value) || -1;
|
|
|
+ // add max capacity
|
|
|
+ field.maxCapacity =
|
|
|
+ Number(
|
|
|
+ field.type_params.find(item => item.key === 'max_capacity')?.value
|
|
|
+ ) || -1;
|
|
|
+ // add max length
|
|
|
+ field.maxLength =
|
|
|
+ Number(
|
|
|
+ field.type_params.find(item => item.key === 'max_length')?.value
|
|
|
+ ) || -1;
|
|
|
+
|
|
|
+ // classify fields
|
|
|
+ if (
|
|
|
+ field.data_type === 'BinaryVector' ||
|
|
|
+ field.data_type === 'FloatVector'
|
|
|
+ ) {
|
|
|
+ vectorFields.push(field);
|
|
|
+ } else {
|
|
|
+ scalarFields.push(field);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (field.is_primary_key) {
|
|
|
+ res.schema.primaryField = field;
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ // add extra data to schema
|
|
|
+ res.schema.hasVectorIndex = vectorFields.some(v => v.index);
|
|
|
+ res.schema.scalarFields = scalarFields;
|
|
|
+ res.schema.vectorFields = vectorFields;
|
|
|
+
|
|
|
return res;
|
|
|
}
|
|
|
|
|
@@ -122,7 +184,7 @@ export class CollectionsService {
|
|
|
);
|
|
|
count = collectionStatisticsRes.data.row_count;
|
|
|
}
|
|
|
- return count;
|
|
|
+ return { rowCount: Number(count) } as CountObject;
|
|
|
}
|
|
|
|
|
|
async insert(clientId: string, data: InsertReq) {
|
|
@@ -194,94 +256,138 @@ export class CollectionsService {
|
|
|
return res;
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * Get all collections meta data
|
|
|
- * @returns {id:string, collection_name:string, schema:Field[], autoID:boolean, rowCount: number, consistency_level:string}
|
|
|
- */
|
|
|
- async getAllCollections(
|
|
|
+ // get single collection details
|
|
|
+ async getCollection(
|
|
|
clientId: string,
|
|
|
- collections?: {
|
|
|
- data: { name: string }[];
|
|
|
+ collection: CollectionData,
|
|
|
+ loadCollection: CollectionData,
|
|
|
+ lazy: boolean = false
|
|
|
+ ) {
|
|
|
+ if (lazy) {
|
|
|
+ return {
|
|
|
+ id: collection.id,
|
|
|
+ collection_name: collection.name,
|
|
|
+ createdTime: Number(collection.timestamp),
|
|
|
+ schema: undefined,
|
|
|
+ rowCount: undefined,
|
|
|
+ aliases: undefined,
|
|
|
+ description: undefined,
|
|
|
+ autoID: undefined,
|
|
|
+ loadedPercentage: undefined,
|
|
|
+ consistency_level: undefined,
|
|
|
+ replicas: undefined,
|
|
|
+ loaded: undefined,
|
|
|
+ } as CollectionLazyObject;
|
|
|
}
|
|
|
- ): Promise<CollectionData[]> {
|
|
|
- const data: CollectionData[] = [];
|
|
|
- const res = collections || (await this.getCollections(clientId));
|
|
|
- const loadedCollections = await this.getCollections(clientId, {
|
|
|
- type: ShowCollectionsType.Loaded,
|
|
|
+ // get collection schema and properties
|
|
|
+ const collectionInfo = await this.describeCollection(clientId, {
|
|
|
+ collection_name: collection.name,
|
|
|
});
|
|
|
- if (res.data.length > 0) {
|
|
|
- for (const item of res.data) {
|
|
|
- const { name } = item;
|
|
|
|
|
|
- // get collection schema and properties
|
|
|
- const collectionInfo = await this.describeCollection(clientId, {
|
|
|
- collection_name: name,
|
|
|
- });
|
|
|
+ // get collection statistic data
|
|
|
+ const collectionStatisticsRes = await this.getCollectionStatistics(
|
|
|
+ clientId,
|
|
|
+ {
|
|
|
+ collection_name: collection.name,
|
|
|
+ }
|
|
|
+ );
|
|
|
+ // extract autoID
|
|
|
+ const autoID = collectionInfo.schema.fields.find(
|
|
|
+ v => v.is_primary_key === true
|
|
|
+ )?.autoID;
|
|
|
|
|
|
- // get collection statistic data
|
|
|
- const collectionStatisticsRes = await this.getCollectionStatistics(
|
|
|
- clientId,
|
|
|
- {
|
|
|
- collection_name: name,
|
|
|
- }
|
|
|
- );
|
|
|
+ // get replica info
|
|
|
+ let replicas;
|
|
|
+ try {
|
|
|
+ replicas = loadCollection
|
|
|
+ ? await this.getReplicas(clientId, {
|
|
|
+ collectionID: collectionInfo.collectionID,
|
|
|
+ })
|
|
|
+ : replicas;
|
|
|
+ } catch (e) {
|
|
|
+ console.log('ignore getReplica');
|
|
|
+ }
|
|
|
|
|
|
- // get index info for collections
|
|
|
- const indexRes = await this.schemaService.describeIndex(clientId, {
|
|
|
- collection_name: item.name,
|
|
|
- });
|
|
|
+ // loading info
|
|
|
+ const loadedPercentage = !loadCollection
|
|
|
+ ? '-1'
|
|
|
+ : loadCollection.loadedPercentage;
|
|
|
+
|
|
|
+ const status =
|
|
|
+ loadedPercentage === '-1'
|
|
|
+ ? LOADING_STATE.UNLOADED
|
|
|
+ : loadedPercentage === '100'
|
|
|
+ ? LOADING_STATE.LOADED
|
|
|
+ : LOADING_STATE.LOADING;
|
|
|
+
|
|
|
+ return {
|
|
|
+ collection_name: collection.name,
|
|
|
+ schema: collectionInfo.schema,
|
|
|
+ rowCount: Number(collectionStatisticsRes.data.row_count),
|
|
|
+ createdTime: parseInt(collectionInfo.created_utc_timestamp, 10),
|
|
|
+ aliases: collectionInfo.aliases,
|
|
|
+ description: collectionInfo.schema.description,
|
|
|
+ autoID,
|
|
|
+ id: collectionInfo.collectionID,
|
|
|
+ loadedPercentage,
|
|
|
+ consistency_level: collectionInfo.consistency_level,
|
|
|
+ replicas: replicas && replicas.replicas,
|
|
|
+ loaded: status === LOADING_STATE.LOADED,
|
|
|
+ status,
|
|
|
+ };
|
|
|
+ }
|
|
|
|
|
|
- // extract autoID
|
|
|
- const autoID = collectionInfo.schema.fields.find(
|
|
|
- v => v.is_primary_key === true
|
|
|
- )?.autoID;
|
|
|
+ async getAllCollections(
|
|
|
+ clientId: string,
|
|
|
+ collectionName?: string
|
|
|
+ ): Promise<CollectionObject[]> {
|
|
|
+ // get all collections(name, timestamp, id)
|
|
|
+ const allCollections = await this.showCollections(clientId);
|
|
|
+ // get all loaded collection
|
|
|
+ const loadedCollections = await this.showCollections(clientId, {
|
|
|
+ type: ShowCollectionsType.Loaded,
|
|
|
+ });
|
|
|
|
|
|
- // if it is loaded
|
|
|
- const loadCollection = loadedCollections.data.find(
|
|
|
- v => v.name === name
|
|
|
- );
|
|
|
+ // data container
|
|
|
+ const data: CollectionObject[] = [];
|
|
|
+ // sort by created time
|
|
|
+ allCollections.data.sort(
|
|
|
+ (a, b) => Number(b.timestamp) - Number(a.timestamp)
|
|
|
+ );
|
|
|
|
|
|
- // loading info
|
|
|
- const loadedPercentage = !loadCollection
|
|
|
- ? '-1'
|
|
|
- : loadCollection.loadedPercentage;
|
|
|
-
|
|
|
- // get replica info
|
|
|
- let replicas;
|
|
|
- try {
|
|
|
- replicas = loadCollection
|
|
|
- ? await this.getReplicas(clientId, {
|
|
|
- collectionID: collectionInfo.collectionID,
|
|
|
- })
|
|
|
- : replicas;
|
|
|
- } catch (e) {
|
|
|
- console.log('ignore getReplica');
|
|
|
- }
|
|
|
+ // get single collection details
|
|
|
+ const targetCollections = allCollections.data.find(
|
|
|
+ d => d.name === collectionName
|
|
|
+ );
|
|
|
+ if (targetCollections) {
|
|
|
+ const res = await this.getCollection(
|
|
|
+ clientId,
|
|
|
+ targetCollections,
|
|
|
+ loadedCollections.data.find(v => v.name === targetCollections.name),
|
|
|
+ false
|
|
|
+ );
|
|
|
+ return [res];
|
|
|
+ }
|
|
|
|
|
|
- data.push({
|
|
|
- aliases: collectionInfo.aliases,
|
|
|
- collection_name: name,
|
|
|
- schema: collectionInfo.schema,
|
|
|
- description: collectionInfo.schema.description,
|
|
|
- autoID,
|
|
|
- rowCount: Number(collectionStatisticsRes.data.row_count),
|
|
|
- id: collectionInfo.collectionID,
|
|
|
- loadedPercentage,
|
|
|
- createdTime: parseInt(collectionInfo.created_utc_timestamp, 10),
|
|
|
- index_descriptions: indexRes.index_descriptions,
|
|
|
- consistency_level: collectionInfo.consistency_level,
|
|
|
- replicas: replicas && replicas.replicas,
|
|
|
- });
|
|
|
- }
|
|
|
+ // get all collection details
|
|
|
+ for (let i = 0; i < allCollections.data.length; i++) {
|
|
|
+ const collection = allCollections.data[i];
|
|
|
+ data.push(
|
|
|
+ await this.getCollection(
|
|
|
+ clientId,
|
|
|
+ collection,
|
|
|
+ loadedCollections.data.find(v => v.name === collection.name),
|
|
|
+ false
|
|
|
+ )
|
|
|
+ );
|
|
|
}
|
|
|
- // add default sort - Descending order
|
|
|
- data.sort((a, b) => b.createdTime - a.createdTime);
|
|
|
+
|
|
|
return data;
|
|
|
}
|
|
|
|
|
|
async getLoadedCollections(clientId: string) {
|
|
|
const data = [];
|
|
|
- const res = await this.getCollections(clientId, {
|
|
|
+ const res = await this.showCollections(clientId, {
|
|
|
type: ShowCollectionsType.Loaded,
|
|
|
});
|
|
|
if (res.data.length > 0) {
|
|
@@ -308,8 +414,8 @@ export class CollectionsService {
|
|
|
const data = {
|
|
|
collectionCount: 0,
|
|
|
totalData: 0,
|
|
|
- };
|
|
|
- const res = await this.getCollections(clientId);
|
|
|
+ } as StatisticsObject;
|
|
|
+ const res = await this.showCollections(clientId);
|
|
|
data.collectionCount = res.data.length;
|
|
|
if (res.data.length > 0) {
|
|
|
for (const item of res.data) {
|
|
@@ -326,27 +432,6 @@ export class CollectionsService {
|
|
|
return data;
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * Get all collection index status
|
|
|
- * @returns {collection_name:string, index_descriptions: index_descriptions}[]
|
|
|
- */
|
|
|
- async getCollectionsIndexStatus(clientId: string) {
|
|
|
- const data = [];
|
|
|
- const res = await this.getCollections(clientId);
|
|
|
- if (res.data.length > 0) {
|
|
|
- for (const item of res.data) {
|
|
|
- const indexRes = await this.schemaService.describeIndex(clientId, {
|
|
|
- collection_name: item.name,
|
|
|
- });
|
|
|
- data.push({
|
|
|
- collection_name: item.name,
|
|
|
- index_descriptions: indexRes,
|
|
|
- });
|
|
|
- }
|
|
|
- }
|
|
|
- return data;
|
|
|
- }
|
|
|
-
|
|
|
/**
|
|
|
* Load sample data into collection
|
|
|
*/
|
|
@@ -416,19 +501,19 @@ export class CollectionsService {
|
|
|
}
|
|
|
|
|
|
async duplicateCollection(clientId: string, data: RenameCollectionReq) {
|
|
|
- const collection: any = await this.describeCollection(clientId, {
|
|
|
+ const collection = await this.describeCollection(clientId, {
|
|
|
collection_name: data.collection_name,
|
|
|
});
|
|
|
|
|
|
const createCollectionParams: CreateCollectionReq = {
|
|
|
collection_name: data.new_collection_name,
|
|
|
fields: collection.schema.fields.map(convertFieldSchemaToFieldType),
|
|
|
- consistency_level: collection.consistency_level,
|
|
|
- enable_dynamic_field: !!collection.enable_dynamic_field,
|
|
|
+ consistency_level: collection.consistency_level as any,
|
|
|
+ enable_dynamic_field: !!collection.schema.enable_dynamic_field,
|
|
|
};
|
|
|
|
|
|
if (
|
|
|
- collection.schema.fields.some((f: FieldSchema) => f.is_partition_key) &&
|
|
|
+ collection.schema.fields.some(f => f.is_partition_key) &&
|
|
|
collection.num_partitions
|
|
|
) {
|
|
|
createCollectionParams.num_partitions = Number(collection.num_partitions);
|
|
@@ -444,7 +529,8 @@ export class CollectionsService {
|
|
|
|
|
|
const res = await milvusClient.deleteEntities({
|
|
|
collection_name: data.collection_name,
|
|
|
- filter: pkType === 'Int64' ? `${pkField} >= 0` : `${pkField} != ''`,
|
|
|
+ filter:
|
|
|
+ pkType === 'Int64' ? `${pkField} >= ${MIN_INT64}` : `${pkField} != ''`,
|
|
|
});
|
|
|
|
|
|
return res;
|