|
|
@@ -8,6 +8,8 @@
|
|
|
|
|
|
package org.elasticsearch.xpack.vectors.mapper;
|
|
|
|
|
|
+import org.apache.lucene.codecs.KnnVectorsFormat;
|
|
|
+import org.apache.lucene.codecs.lucene90.Lucene90HnswVectorsFormat;
|
|
|
import org.apache.lucene.document.BinaryDocValuesField;
|
|
|
import org.apache.lucene.document.Field;
|
|
|
import org.apache.lucene.document.KnnVectorField;
|
|
|
@@ -16,6 +18,10 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery;
|
|
|
import org.apache.lucene.search.Query;
|
|
|
import org.apache.lucene.util.BytesRef;
|
|
|
import org.elasticsearch.Version;
|
|
|
+import org.elasticsearch.index.mapper.MappingParser;
|
|
|
+import org.elasticsearch.index.mapper.PerFieldKnnVectorsFormatFieldMapper;
|
|
|
+import org.elasticsearch.xcontent.ToXContent;
|
|
|
+import org.elasticsearch.xcontent.XContentBuilder;
|
|
|
import org.elasticsearch.xcontent.XContentParser.Token;
|
|
|
import org.elasticsearch.common.xcontent.support.XContentMapValues;
|
|
|
import org.elasticsearch.index.fielddata.IndexFieldData;
|
|
|
@@ -40,6 +46,7 @@ import java.nio.ByteBuffer;
|
|
|
import java.time.ZoneId;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
+import java.util.Objects;
|
|
|
import java.util.function.Supplier;
|
|
|
|
|
|
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
|
|
|
@@ -47,7 +54,7 @@ import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpect
|
|
|
/**
|
|
|
* A {@link FieldMapper} for indexing a dense vector of floats.
|
|
|
*/
|
|
|
-public class DenseVectorFieldMapper extends FieldMapper {
|
|
|
+public class DenseVectorFieldMapper extends FieldMapper implements PerFieldKnnVectorsFormatFieldMapper {
|
|
|
|
|
|
public static final String CONTENT_TYPE = "dense_vector";
|
|
|
public static short MAX_DIMS_COUNT = 2048; //maximum allowed number of dimensions
|
|
|
@@ -73,6 +80,8 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|
|
private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, false);
|
|
|
private final Parameter<VectorSimilarity> similarity = Parameter.enumParam(
|
|
|
"similarity", false, m -> toType(m).similarity, null, VectorSimilarity.class);
|
|
|
+ private final Parameter<IndexOptions> indexOptions = new Parameter<>("index_options", false, () -> null,
|
|
|
+ (n, c, o) -> o == null ? null : parseIndexOptions(n, o), m -> toType(m).indexOptions);
|
|
|
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
|
|
|
|
|
|
final Version indexVersionCreated;
|
|
|
@@ -84,11 +93,13 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|
|
this.indexed.requiresParameters(similarity);
|
|
|
this.similarity.setSerializerCheck((id, ic, v) -> v != null);
|
|
|
this.similarity.requiresParameters(indexed);
|
|
|
+ this.indexOptions.requiresParameters(indexed);
|
|
|
+ this.indexOptions.setSerializerCheck((id, ic, v) -> v != null);
|
|
|
}
|
|
|
|
|
|
@Override
|
|
|
protected List<Parameter<?>> getParameters() {
|
|
|
- return List.of(dims, indexed, similarity, meta);
|
|
|
+ return List.of(dims, indexed, similarity, indexOptions, meta);
|
|
|
}
|
|
|
|
|
|
@Override
|
|
|
@@ -100,6 +111,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|
|
dims.getValue(),
|
|
|
indexed.getValue(),
|
|
|
similarity.getValue(),
|
|
|
+ indexOptions.getValue(),
|
|
|
indexVersionCreated,
|
|
|
multiFieldsBuilder.build(this, context),
|
|
|
copyTo.build());
|
|
|
@@ -116,6 +128,67 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ private abstract static class IndexOptions implements ToXContent {
|
|
|
+ final String type;
|
|
|
+ IndexOptions(String type) {
|
|
|
+ this.type = type;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static class HnswIndexOptions extends IndexOptions {
|
|
|
+ private final int m;
|
|
|
+ private final int efConstruction;
|
|
|
+
|
|
|
+ static IndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOptionsMap) {
|
|
|
+ Object mNode = indexOptionsMap.remove("m");
|
|
|
+ Object efConstructionNode = indexOptionsMap.remove("ef_construction");
|
|
|
+ if (mNode == null) {
|
|
|
+ throw new MapperParsingException("[index_options] of type [hnsw] requires field [m] to be configured");
|
|
|
+ }
|
|
|
+ if (efConstructionNode == null) {
|
|
|
+ throw new MapperParsingException("[index_options] of type [hnsw] requires field [ef_construction] to be configured");
|
|
|
+ }
|
|
|
+ int m = XContentMapValues.nodeIntegerValue(mNode);
|
|
|
+ int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode);
|
|
|
+ MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
|
|
|
+ return new HnswIndexOptions(m, efConstruction);
|
|
|
+ }
|
|
|
+
|
|
|
+ private HnswIndexOptions(int m, int efConstruction) {
|
|
|
+ super("hnsw");
|
|
|
+ this.m = m;
|
|
|
+ this.efConstruction = efConstruction;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
|
|
|
+ builder.startObject();
|
|
|
+ builder.field("type", type);
|
|
|
+ builder.field("m", m);
|
|
|
+ builder.field("ef_construction", efConstruction);
|
|
|
+ builder.endObject();
|
|
|
+ return builder;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public boolean equals(Object o) {
|
|
|
+ if (this == o) return true;
|
|
|
+ if (o == null || getClass() != o.getClass()) return false;
|
|
|
+ HnswIndexOptions that = (HnswIndexOptions) o;
|
|
|
+ return m == that.m && efConstruction == that.efConstruction;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public int hashCode() {
|
|
|
+ return Objects.hash(type, m, efConstruction);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public String toString() {
|
|
|
+ return "{type=" + type + ", m=" + m + ", ef_construction=" + efConstruction + " }";
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
public static final TypeParser PARSER
|
|
|
= new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated()), notInMultiFields(CONTENT_TYPE));
|
|
|
|
|
|
@@ -185,15 +258,17 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|
|
private final int dims;
|
|
|
private final boolean indexed;
|
|
|
private final VectorSimilarity similarity;
|
|
|
+ private final IndexOptions indexOptions;
|
|
|
private final Version indexCreatedVersion;
|
|
|
|
|
|
- private DenseVectorFieldMapper(String simpleName, MappedFieldType mappedFieldType, int dims,
|
|
|
- boolean indexed, VectorSimilarity similarity,
|
|
|
+ private DenseVectorFieldMapper(String simpleName, MappedFieldType mappedFieldType, int dims, boolean indexed,
|
|
|
+ VectorSimilarity similarity, IndexOptions indexOptions,
|
|
|
Version indexCreatedVersion, MultiFields multiFields, CopyTo copyTo) {
|
|
|
super(simpleName, mappedFieldType, multiFields, copyTo);
|
|
|
this.dims = dims;
|
|
|
this.indexed = indexed;
|
|
|
this.similarity = similarity;
|
|
|
+ this.indexOptions = indexOptions;
|
|
|
this.indexCreatedVersion = indexCreatedVersion;
|
|
|
}
|
|
|
|
|
|
@@ -289,4 +364,29 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|
|
public FieldMapper.Builder getMergeBuilder() {
|
|
|
return new Builder(simpleName(), indexCreatedVersion).init(this);
|
|
|
}
|
|
|
+
|
|
|
+ private static IndexOptions parseIndexOptions(String fieldName, Object propNode) {
|
|
|
+ @SuppressWarnings("unchecked")
|
|
|
+ Map<String, ?> indexOptionsMap = (Map<String, ?>) propNode;
|
|
|
+ Object typeNode = indexOptionsMap.remove("type");
|
|
|
+ if (typeNode == null) {
|
|
|
+ throw new MapperParsingException("[index_options] requires field [type] to be configured");
|
|
|
+ }
|
|
|
+ String type = XContentMapValues.nodeStringValue(typeNode);
|
|
|
+ if (type.equals("hnsw")) {
|
|
|
+ return HnswIndexOptions.parseIndexOptions(fieldName, indexOptionsMap);
|
|
|
+ } else {
|
|
|
+ throw new MapperParsingException("Unknown vector index options type [" + type + "] for field [" + fieldName + "]");
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public KnnVectorsFormat getKnnVectorsFormatForField() {
|
|
|
+ if (indexOptions == null) {
|
|
|
+ return null; // use default format
|
|
|
+ } else {
|
|
|
+ HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions;
|
|
|
+ return new Lucene90HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|
Mayya Sharipova