/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package io.milvus.v1; import io.milvus.common.utils.Float16Utils; import io.milvus.param.R; import org.tensorflow.Tensor; import org.tensorflow.ndarray.Shape; import org.tensorflow.ndarray.buffer.ByteDataBuffer; import org.tensorflow.ndarray.buffer.DataBuffers; import org.tensorflow.types.TBfloat16; import org.tensorflow.types.TFloat16; import java.nio.ByteBuffer; import java.util.*; public class CommonUtils { public static void handleResponseStatus(R r) { if (r.getStatus() != R.Status.Success.getCode()) { throw new RuntimeException(r.getMessage()); } } public static List generateFloatVector(int dimension) { Random ran = new Random(); List vector = new ArrayList<>(); for (int i = 0; i < dimension; ++i) { vector.add(ran.nextFloat()); } return vector; } public static List generateFloatVector(int dimension, Float initValue) { List vector = new ArrayList<>(); for (int i = 0; i < dimension; ++i) { vector.add(initValue); } return vector; } public static List> generateFloatVectors(int dimension, int count) { List> vectors = new ArrayList<>(); for (int n = 0; n < count; ++n) { List vector = generateFloatVector(dimension); vectors.add(vector); } return vectors; } public static List> generateFixFloatVectors(int dimension, int count) { List> vectors = new ArrayList<>(); for (int n = 0; n < count; ++n) { List vector = generateFloatVector(dimension, (float)n); vectors.add(vector); } return vectors; } ///////////////////////////////////////////////////////////////////////////////////////////////////// public static ByteBuffer generateBinaryVector(int dimension) { Random ran = new Random(); int byteCount = dimension / 8; // binary vector doesn't care endian since each byte is independent ByteBuffer vector = ByteBuffer.allocate(byteCount); for (int i = 0; i < byteCount; ++i) { vector.put((byte) ran.nextInt(Byte.MAX_VALUE)); } return vector; } public static List generateBinaryVectors(int dimension, int count) { List vectors = new ArrayList<>(); for (int n = 0; n < count; ++n) { ByteBuffer vector = generateBinaryVector(dimension); vectors.add(vector); } return vectors; } ///////////////////////////////////////////////////////////////////////////////////////////////////// public static TBfloat16 genTensorflowBF16Vector(int dimension) { Random ran = new Random(); float[] array = new float[dimension]; for (int n = 0; n < dimension; ++n) { array[n] = ran.nextFloat(); } return TBfloat16.vectorOf(array); } public static List genTensorflowBF16Vectors(int dimension, int count) { List vectors = new ArrayList<>(); for (int n = 0; n < count; ++n) { TBfloat16 vector = genTensorflowBF16Vector(dimension); vectors.add(vector); } return vectors; } public static ByteBuffer encodeTensorBF16Vector(TBfloat16 vector) { ByteDataBuffer tensorBuf = vector.asRawTensor().data(); ByteBuffer buf = ByteBuffer.allocate((int)tensorBuf.size()); for (long i = 0; i < tensorBuf.size(); i++) { buf.put(tensorBuf.getByte(i)); } return buf; } public static List encodeTensorBF16Vectors(List vectors) { List buffers = new ArrayList<>(); for (TBfloat16 tf : vectors) { ByteBuffer bf = encodeTensorBF16Vector(tf); buffers.add(bf); } return buffers; } public static TBfloat16 decodeTensorBF16Vector(ByteBuffer buf) { if (buf.limit()%2 != 0) { return null; } int dim = buf.limit()/2; ByteDataBuffer bf = DataBuffers.of(buf.array()); return Tensor.of(TBfloat16.class, Shape.of(dim), bf); } public static TFloat16 genTensorflowFP16Vector(int dimension) { Random ran = new Random(); float[] array = new float[dimension]; for (int n = 0; n < dimension; ++n) { array[n] = ran.nextFloat(); } return TFloat16.vectorOf(array); } public static List genTensorflowFP16Vectors(int dimension, int count) { List vectors = new ArrayList<>(); for (int n = 0; n < count; ++n) { TFloat16 vector = genTensorflowFP16Vector(dimension); vectors.add(vector); } return vectors; } public static ByteBuffer encodeTensorFP16Vector(TFloat16 vector) { ByteDataBuffer tensorBuf = vector.asRawTensor().data(); ByteBuffer buf = ByteBuffer.allocate((int)tensorBuf.size()); for (long i = 0; i < tensorBuf.size(); i++) { buf.put(tensorBuf.getByte(i)); } return buf; } public static List encodeTensorFP16Vectors(List vectors) { List buffers = new ArrayList<>(); for (TFloat16 tf : vectors) { ByteBuffer bf = encodeTensorFP16Vector(tf); buffers.add(bf); } return buffers; } public static TFloat16 decodeTensorFP16Vector(ByteBuffer buf) { if (buf.limit()%2 != 0) { return null; } int dim = buf.limit()/2; ByteDataBuffer bf = DataBuffers.of(buf.array()); return Tensor.of(TFloat16.class, Shape.of(dim), bf); } ///////////////////////////////////////////////////////////////////////////////////////////////////// public static ByteBuffer encodeFloat16Vector(List originVector, boolean bfloat16) { if (bfloat16) { return Float16Utils.f32VectorToBf16Buffer(originVector); } else { return Float16Utils.f32VectorToFp16Buffer(originVector); } } public static List decodeFloat16Vector(ByteBuffer buf, boolean bfloat16) { if (bfloat16) { return Float16Utils.bf16BufferToVector(buf); } else { return Float16Utils.fp16BufferToVector(buf); } } public static List encodeFloat16Vectors(List> originVectors, boolean bfloat16) { List vectors = new ArrayList<>(); for (List originVector : originVectors) { if (bfloat16) { vectors.add(Float16Utils.f32VectorToBf16Buffer(originVector)); } else { vectors.add(Float16Utils.f32VectorToFp16Buffer(originVector)); } } return vectors; } public static ByteBuffer generateFloat16Vector(int dimension, boolean bfloat16) { List originalVector = generateFloatVector(dimension); return encodeFloat16Vector(originalVector, bfloat16); } public static List generateFloat16Vectors(int dimension, int count, boolean bfloat16) { List vectors = new ArrayList<>(); for (int i = 0; i < count; i++) { ByteBuffer buf = generateFloat16Vector(dimension, bfloat16); vectors.add((buf)); } return vectors; } ///////////////////////////////////////////////////////////////////////////////////////////////////// public static SortedMap generateSparseVector() { Random ran = new Random(); SortedMap sparse = new TreeMap<>(); int dim = ran.nextInt(10) + 10; for (int i = 0; i < dim; ++i) { sparse.put((long)ran.nextInt(1000000), ran.nextFloat()); } return sparse; } public static List> generateSparseVectors(int count) { List> vectors = new ArrayList<>(); for (int n = 0; n < count; ++n) { SortedMap sparse = generateSparseVector(); vectors.add(sparse); } return vectors; } }