123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488 |
- /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
- package io.milvus;
- import io.milvus.client.MilvusServiceClient;
- import io.milvus.grpc.*;
- import io.milvus.param.*;
- import io.milvus.param.collection.*;
- import io.milvus.param.dml.*;
- import io.milvus.param.index.*;
- import io.milvus.param.partition.*;
- import io.milvus.Response.*;
- import java.nio.ByteBuffer;
- import java.util.*;
- public class GeneralExample {
- private static final MilvusServiceClient milvusClient;
- static {
- ConnectParam connectParam = ConnectParam.newBuilder()
- .withHost("localhost")
- .withPort(19530)
- .build();
- milvusClient = new MilvusServiceClient(connectParam);
- }
- private static final String COLLECTION_NAME = "TEST";
- private static final String ID_FIELD = "userID";
- private static final String VECTOR_FIELD = "userFace";
- private static final Integer VECTOR_DIM = 64;
- private static final String AGE_FIELD = "userAge";
- private static final String PROFILE_FIELD = "userProfile";
- private static final Integer BINARY_DIM = 128;
- private static final IndexType INDEX_TYPE = IndexType.IVF_FLAT;
- private static final String INDEX_PARAM = "{\"nlist\":128}";
- private static final Integer SEARCH_K = 5;
- private static final String SEARCH_PARAM = "{\"nprobe\":10}";
- private R<RpcStatus> createCollection() {
- System.out.println("========== createCollection() ==========");
- FieldType fieldType1 = FieldType.newBuilder()
- .withName(ID_FIELD)
- .withDescription("user identification")
- .withDataType(DataType.Int64)
- .withPrimaryKey(true)
- .withAutoID(true)
- .build();
- FieldType fieldType2 = FieldType.newBuilder()
- .withName(VECTOR_FIELD)
- .withDescription("face embedding")
- .withDataType(DataType.FloatVector)
- .withDimension(VECTOR_DIM)
- .build();
- FieldType fieldType3 = FieldType.newBuilder()
- .withName(AGE_FIELD)
- .withDescription("user age")
- .withDataType(DataType.Int8)
- .build();
- FieldType fieldType4 = FieldType.newBuilder()
- .withName(PROFILE_FIELD)
- .withDescription("user profile")
- .withDataType(DataType.BinaryVector)
- .withDimension(BINARY_DIM)
- .build();
- CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withDescription("customer info")
- .withShardsNum(2)
- .addFieldType(fieldType1)
- .addFieldType(fieldType2)
- .addFieldType(fieldType3)
- .addFieldType(fieldType4)
- .build();
- R<RpcStatus> response = milvusClient.createCollection(createCollectionReq);
- System.out.println(response);
- return response;
- }
- private R<RpcStatus> dropCollection() {
- System.out.println("========== dropCollection() ==========");
- R<RpcStatus> response = milvusClient.dropCollection(DropCollectionParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .build());
- System.out.println(response);
- return response;
- }
- private R<Boolean> hasCollection() {
- System.out.println("========== hasCollection() ==========");
- R<Boolean> response = milvusClient.hasCollection(HasCollectionParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .build());
- System.out.println(response);
- return response;
- }
- private R<RpcStatus> loadCollection() {
- System.out.println("========== loadCollection() ==========");
- R<RpcStatus> response = milvusClient.loadCollection(LoadCollectionParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .build());
- System.out.println(response);
- return response;
- }
- private R<RpcStatus> releaseCollection() {
- System.out.println("========== releaseCollection() ==========");
- R<RpcStatus> response = milvusClient.releaseCollection(ReleaseCollectionParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .build());
- System.out.println(response);
- return response;
- }
- private R<DescribeCollectionResponse> describeCollection() {
- System.out.println("========== describeCollection() ==========");
- R<DescribeCollectionResponse> response = milvusClient.describeCollection(DescribeCollectionParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .build());
- System.out.println(response);
- return response;
- }
- private R<GetCollectionStatisticsResponse> getCollectionStatistics() {
- System.out.println("========== getCollectionStatistics() ==========");
- R<GetCollectionStatisticsResponse> response = milvusClient.getCollectionStatistics(
- GetCollectionStatisticsParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .build());
- GetCollStatResponseWrapper wrapper = new GetCollStatResponseWrapper(response.getData());
- System.out.println("Collection row count: " + wrapper.GetRowCount());
- return response;
- }
- private R<ShowCollectionsResponse> showCollections() {
- System.out.println("========== showCollections() ==========");
- R<ShowCollectionsResponse> response = milvusClient.showCollections(ShowCollectionsParam.newBuilder()
- .build());
- System.out.println(response);
- return response;
- }
- private R<RpcStatus> createPartition(String partitionName) {
- System.out.println("========== createPartition() ==========");
- R<RpcStatus> response = milvusClient.createPartition(CreatePartitionParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withPartitionName(partitionName)
- .build());
- System.out.println(response);
- return response;
- }
- private R<RpcStatus> dropPartition(String partitionName) {
- System.out.println("========== dropPartition() ==========");
- R<RpcStatus> response = milvusClient.dropPartition(DropPartitionParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withPartitionName(partitionName)
- .build());
- System.out.println(response);
- return response;
- }
- private R<Boolean> hasPartition(String partitionName) {
- System.out.println("========== hasPartition() ==========");
- R<Boolean> response = milvusClient.hasPartition(HasPartitionParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withPartitionName(partitionName)
- .build());
- System.out.println(response);
- return response;
- }
- private R<RpcStatus> releasePartition(String partitionName) {
- System.out.println("========== releasePartition() ==========");
- R<RpcStatus> response = milvusClient.releasePartitions(ReleasePartitionsParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .addPartitionName(partitionName)
- .build());
- System.out.println(response);
- return response;
- }
- private R<ShowPartitionsResponse> showPartitions() {
- System.out.println("========== showPartitions() ==========");
- R<ShowPartitionsResponse> response = milvusClient.showPartitions(ShowPartitionsParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .build());
- System.out.println(response);
- return response;
- }
- private R<RpcStatus> createIndex() {
- System.out.println("========== createIndex() ==========");
- R<RpcStatus> response = milvusClient.createIndex(CreateIndexParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withFieldName(VECTOR_FIELD)
- .withIndexType(INDEX_TYPE)
- .withMetricType(MetricType.L2)
- .withExtraParam(INDEX_PARAM)
- .withSyncMode(Boolean.TRUE)
- .build());
- System.out.println(response);
- return response;
- }
- private R<RpcStatus> dropIndex() {
- System.out.println("========== dropIndex() ==========");
- R<RpcStatus> response = milvusClient.dropIndex(DropIndexParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withFieldName(VECTOR_FIELD)
- .build());
- System.out.println(response);
- return response;
- }
- private R<DescribeIndexResponse> describeIndex() {
- System.out.println("========== describeIndex() ==========");
- R<DescribeIndexResponse> response = milvusClient.describeIndex(DescribeIndexParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withFieldName(VECTOR_FIELD)
- .build());
- System.out.println(response);
- return response;
- }
- private R<GetIndexStateResponse> getIndexState() {
- System.out.println("========== getIndexState() ==========");
- R<GetIndexStateResponse> response = milvusClient.getIndexState(GetIndexStateParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withFieldName(VECTOR_FIELD)
- .build());
- System.out.println(response);
- return response;
- }
- private R<GetIndexBuildProgressResponse> getIndexBuildProgress() {
- System.out.println("========== getIndexBuildProgress() ==========");
- R<GetIndexBuildProgressResponse> response = milvusClient.getIndexBuildProgress(
- GetIndexBuildProgressParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .build());
- System.out.println(response);
- return response;
- }
- private R<MutationResult> delete(String partitionName, String expr) {
- System.out.println("========== delete() ==========");
- DeleteParam build = DeleteParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withPartitionName(partitionName)
- .withExpr(expr)
- .build();
- R<MutationResult> response = milvusClient.delete(build);
- System.out.println(response.getData());
- return response;
- }
- private R<SearchResults> searchFace(String expr) {
- System.out.println("========== searchFace() ==========");
- List<String> outFields = Collections.singletonList(AGE_FIELD);
- List<List<Float>> vectors = generateFloatVectors(5);
- SearchParam searchParam = SearchParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withMetricType(MetricType.L2)
- .withOutFields(outFields)
- .withTopK(SEARCH_K)
- .withVectors(vectors)
- .withVectorFieldName(VECTOR_FIELD)
- .withExpr(expr)
- .withParams(SEARCH_PARAM)
- .build();
- R<SearchResults> response = milvusClient.search(searchParam);
- SearchResultsWrapper wrapper = new SearchResultsWrapper(response.getData().getResults());
- for (int i = 0; i < vectors.size(); ++i) {
- System.out.println("Search result of No." + i);
- List<SearchResultsWrapper.IDScore> scores = wrapper.GetIDScore(i);
- System.out.println(scores);
- }
- System.out.println(wrapper.GetFieldData(AGE_FIELD).getFieldData());
- return response;
- }
- private R<SearchResults> searchProfile(String expr) {
- System.out.println("========== searchProfile() ==========");
- List<String> outFields = Collections.singletonList(AGE_FIELD);
- List<ByteBuffer> vectors = generateBinaryVectors(5);
- SearchParam searchParam = SearchParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withMetricType(MetricType.HAMMING)
- .withOutFields(outFields)
- .withTopK(SEARCH_K)
- .withVectors(vectors)
- .withVectorFieldName(PROFILE_FIELD)
- .withExpr(expr)
- .withParams(SEARCH_PARAM)
- .build();
- R<SearchResults> response = milvusClient.search(searchParam);
- SearchResultsWrapper wrapper = new SearchResultsWrapper(response.getData().getResults());
- for (int i = 0; i < vectors.size(); ++i) {
- System.out.println("Search result of No." + i);
- List<SearchResultsWrapper.IDScore> scores = wrapper.GetIDScore(i);
- System.out.println(scores);
- }
- System.out.println(wrapper.GetFieldData(AGE_FIELD).getFieldData());
- return response;
- }
- private R<CalcDistanceResults> calDistance() {
- System.out.println("========== calDistance() ==========");
- Random ran=new Random();
- List<Float> vector1 = new ArrayList<>();
- List<Float> vector2 = new ArrayList<>();
- for (int d = 0; d < VECTOR_DIM; ++d) {
- vector1.add(ran.nextFloat());
- vector2.add(ran.nextFloat());
- }
- CalcDistanceParam calcDistanceParam = CalcDistanceParam.newBuilder()
- .withVectorsLeft(Collections.singletonList(vector1))
- .withVectorsRight(Collections.singletonList(vector2))
- .withMetricType(MetricType.L2)
- .build();
- R<CalcDistanceResults> response = milvusClient.calcDistance(calcDistanceParam);
- System.out.println(response);
- return response;
- }
- private R<QueryResults> query(String expr) {
- System.out.println("========== query() ==========");
- List<String> fields = Arrays.asList(ID_FIELD, AGE_FIELD);
- QueryParam test = QueryParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withExpr(expr)
- .withOutFields(fields)
- .build();
- R<QueryResults> response = milvusClient.query(test);
- QueryResultsWrapper wrapper = new QueryResultsWrapper(response.getData());
- System.out.println(ID_FIELD + ":" + wrapper.getFieldWrapper(ID_FIELD).getFieldData().toString());
- System.out.println(AGE_FIELD + ":" + wrapper.getFieldWrapper(AGE_FIELD).getFieldData().toString());
- System.out.println("Query row count: " + wrapper.getFieldWrapper(ID_FIELD).getRowCount());
- return response;
- }
- private R<MutationResult> insert(String partitionName, int count) {
- System.out.println("========== insert() ==========");
- List<List<Float>> vectors = generateFloatVectors(count);
- List<ByteBuffer> profiles = generateBinaryVectors(count);
- Random ran = new Random();
- List<Integer> ages = new ArrayList<>();
- for (long i = 0L; i < count; ++i) {
- ages.add(ran.nextInt(99));
- }
- List<InsertParam.Field> fields = new ArrayList<>();
- fields.add(new InsertParam.Field(VECTOR_FIELD, DataType.FloatVector, vectors));
- fields.add(new InsertParam.Field(PROFILE_FIELD, DataType.BinaryVector, profiles));
- fields.add(new InsertParam.Field(AGE_FIELD, DataType.Int8, ages));
- InsertParam insertParam = InsertParam.newBuilder()
- .withCollectionName(COLLECTION_NAME)
- .withPartitionName(partitionName)
- .withFields(fields)
- .build();
- return milvusClient.insert(insertParam);
- }
- private List<List<Float>> generateFloatVectors(int count) {
- Random ran = new Random();
- List<List<Float>> vectors = new ArrayList<>();
- for (int n = 0; n < count; ++n) {
- List<Float> vector = new ArrayList<>();
- for (int i = 0; i < VECTOR_DIM; ++i) {
- vector.add(ran.nextFloat());
- }
- vectors.add(vector);
- }
- return vectors;
- }
- private List<ByteBuffer> generateBinaryVectors(int count) {
- Random ran = new Random();
- List<ByteBuffer> vectors = new ArrayList<>();
- int byteCount = BINARY_DIM/8;
- for (int n = 0; n < count; ++n) {
- ByteBuffer vector = ByteBuffer.allocate(byteCount);
- for (int i = 0; i < byteCount; ++i) {
- vector.put((byte)ran.nextInt(Byte.MAX_VALUE));
- }
- vectors.add(vector);
- }
- return vectors;
- }
- public static void main(String[] args) {
- GeneralExample example = new GeneralExample();
- example.dropCollection();
- example.createCollection();
- example.hasCollection();
- example.describeCollection();
- example.showCollections();
- example.loadCollection();
- final String partitionName = "p1";
- example.createPartition(partitionName);
- example.hasPartition(partitionName);
- example.showPartitions();
- final int row_count = 10000;
- List<Long> deleteIds = new ArrayList<>();
- Random ran = new Random();
- for (int i = 0; i < 100; ++i) {
- R<MutationResult> result = example.insert(partitionName, row_count);
- InsertResultWrapper wrapper = new InsertResultWrapper(result.getData());
- List<Long> ids = wrapper.getLongIDs();
- deleteIds.add(ids.get(ran.nextInt(row_count)));
- }
- example.getCollectionStatistics();
- example.createIndex();
- example.describeIndex();
- example.getIndexBuildProgress();
- example.getIndexState();
- String deleteExpr = ID_FIELD + " in " + deleteIds.toString();
- example.delete(partitionName, deleteExpr);
- String queryExpr = AGE_FIELD + " == 60";
- example.query(queryExpr);
- String searchExpr = AGE_FIELD + " > 50";
- example.searchFace(searchExpr);
- searchExpr = AGE_FIELD + " <= 30";
- example.searchProfile(searchExpr);
- example.calDistance();
- example.releasePartition(partitionName);
- example.releaseCollection();
- example.dropPartition(partitionName);
- example.dropIndex();
- example.dropCollection();
- }
- }
|