ES|QL brute force dot_product vector function (#131629)

docs/reference/query-languages/esql/_snippets/functions/description/v_cosine.md

@@ -0,0 +1,6 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+**Description**
+
+Calculates the cosine similarity between two dense_vectors.
+

docs/reference/query-languages/esql/_snippets/functions/description/v_dot_product.md

@@ -0,0 +1,6 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+**Description**
+
+Calculates the dot product between two dense_vectors.
+

docs/reference/query-languages/esql/_snippets/functions/examples/v_cosine.md

@@ -0,0 +1,25 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+**Example**
+
+```esql
+ from colors
+ | where color != "black"
+ | eval similarity = v_cosine(rgb_vector, [0, 255, 255])
+ | sort similarity desc, color asc
+```
+
+| color:text | similarity:double |
+| --- | --- |
+| cyan | 1.0 |
+| teal | 1.0 |
+| turquoise | 0.9890533685684204 |
+| aqua marine | 0.964962363243103 |
+| azure | 0.916246771812439 |
+| lavender | 0.9136701822280884 |
+| mint cream | 0.9122757911682129 |
+| honeydew | 0.9122424125671387 |
+| gainsboro | 0.9082483053207397 |
+| gray | 0.9082483053207397 |
+
+

docs/reference/query-languages/esql/_snippets/functions/examples/v_dot_product.md

@@ -0,0 +1,9 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+**Example**
+
+```esql
+null
+```
+
+

docs/reference/query-languages/esql/_snippets/functions/layout/v_cosine.md

@@ -0,0 +1,27 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+## `V_COSINE` [esql-v_cosine]
+```{applies_to}
+stack: development
+serverless: preview
+```
+
+**Syntax**
+
+:::{image} ../../../images/functions/v_cosine.svg
+:alt: Embedded
+:class: text-center
+:::
+
+
+:::{include} ../parameters/v_cosine.md
+:::
+
+:::{include} ../description/v_cosine.md
+:::
+
+:::{include} ../types/v_cosine.md
+:::
+
+:::{include} ../examples/v_cosine.md
+:::

docs/reference/query-languages/esql/_snippets/functions/layout/v_dot_product.md

@@ -0,0 +1,27 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+## `V_DOT_PRODUCT` [esql-v_dot_product]
+```{applies_to}
+stack: development
+serverless: preview
+```
+
+**Syntax**
+
+:::{image} ../../../images/functions/v_dot_product.svg
+:alt: Embedded
+:class: text-center
+:::
+
+
+:::{include} ../parameters/v_dot_product.md
+:::
+
+:::{include} ../description/v_dot_product.md
+:::
+
+:::{include} ../types/v_dot_product.md
+:::
+
+:::{include} ../examples/v_dot_product.md
+:::

docs/reference/query-languages/esql/_snippets/functions/parameters/v_cosine.md

@@ -0,0 +1,10 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+**Parameters**
+
+`left`
+:   first dense_vector to calculate cosine similarity
+
+`right`
+:   second dense_vector to calculate cosine similarity
+

docs/reference/query-languages/esql/_snippets/functions/parameters/v_dot_product.md

@@ -0,0 +1,10 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+**Parameters**
+
+`left`
+:   first dense_vector to calculate dot product similarity
+
+`right`
+:   second dense_vector to calculate dot product similarity
+

docs/reference/query-languages/esql/images/functions/v_dot_product.svg

@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="480" height="46" viewbox="0 0 480 46"><defs><style type="text/css">.c{fill:none;stroke:#222222;}.k{fill:#000000;font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;font-size:20px;}.s{fill:#e4f4ff;stroke:#222222;}.syn{fill:#8D8D8D;font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;font-size:20px;}</style></defs><path class="c" d="M0 31h5m176 0h10m32 0h10m68 0h10m32 0h10m80 0h10m32 0h5"/><rect class="s" x="5" y="5" width="176" height="36"/><text class="k" x="15" y="31">V_DOT_PRODUCT</text><rect class="s" x="191" y="5" width="32" height="36" rx="7"/><text class="syn" x="201" y="31">(</text><rect class="s" x="233" y="5" width="68" height="36" rx="7"/><text class="k" x="243" y="31">left</text><rect class="s" x="311" y="5" width="32" height="36" rx="7"/><text class="syn" x="321" y="31">,</text><rect class="s" x="353" y="5" width="80" height="36" rx="7"/><text class="k" x="363" y="31">right</text><rect class="s" x="443" y="5" width="32" height="36" rx="7"/><text class="syn" x="453" y="31">)</text></svg>

docs/reference/query-languages/esql/kibana/definition/functions/v_dot_product.json

@@ -0,0 +1,12 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.",
+  "type" : "scalar",
+  "name" : "v_dot_product",
+  "description" : "Calculates the dot product between two dense_vectors.",
+  "signatures" : [ ],
+  "examples" : [
+    null
+  ],
+  "preview" : true,
+  "snapshot_only" : true
+}

docs/reference/query-languages/esql/kibana/docs/functions/v_dot_product.md

@@ -0,0 +1,8 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+### V DOT PRODUCT
+Calculates the dot product between two dense_vectors.
+
+```esql
+null
+```

x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-dot-product.csv-spec

@@ -0,0 +1,90 @@
+ # Tests for dot_product similarity function
+ 
+ similarityWithVectorField
+ required_capability: dot_product_vector_similarity_function
+ 
+// tag::vector-dot-product-similarity[]
+ from colors
+ | eval similarity = v_dot_product(rgb_vector, [0, 255, 255]) 
+ | sort similarity desc, color asc 
+// end::vector-dot-product-similarity[]
+ | limit 10
+ | keep color, similarity
+ ;
+ 
+// tag::vector-dot-product-similarity-result[]
+color:text | similarity:double
+azure      | 65025.5
+cyan       | 65025.5
+white      | 65025.5
+mint cream | 64388.0
+snow       | 63750.5
+honeydew   | 63113.0
+ivory      | 63113.0
+sea shell  | 61583.0
+lavender   | 61200.5
+old lace   | 60563.0
+// end::vector-dot-product-similarity-result[] 
+;
+
+ similarityAsPartOfExpression
+ required_capability: dot_product_vector_similarity_function
+ 
+ from colors
+ | eval score = round((1 + v_dot_product(rgb_vector, [0, 255, 255]) / 2), 3) 
+ | sort score desc, color asc 
+ | limit 10
+ | keep color, score
+ ;
+
+color:text | score:double
+azure      | 32513.75
+cyan       | 32513.75
+white      | 32513.75
+mint cream | 32195.0
+snow       | 31876.25
+honeydew   | 31557.5
+ivory      | 31557.5
+sea shell  | 30792.5
+lavender   | 30601.25
+old lace   | 30282.5
+;
+
+similarityWithLiteralVectors
+required_capability: dot_product_vector_similarity_function
+ 
+row a = 1
+| eval similarity = round(v_dot_product([1, 2, 3], [0, 1, 2]), 3) 
+| keep similarity
+;
+
+similarity:double
+4.5
+;
+
+ similarityWithStats
+ required_capability: dot_product_vector_similarity_function
+ 
+ from colors
+ | eval similarity = round(v_dot_product(rgb_vector, [0, 255, 255]), 3) 
+ | stats avg = round(avg(similarity), 3), min = min(similarity), max = max(similarity)
+ ;
+
+avg:double | min:double | max:double
+39519.017  | 0.5        | 65025.5
+;
+
+# TODO Need to implement a conversion function to convert a non-foldable row to a dense_vector
+similarityWithRow-Ignore
+required_capability: dot_product_vector_similarity_function
+ 
+row vector = [1, 2, 3] 
+| eval similarity = round(v_dot_product(vector, [0, 1, 2]), 3) 
+| sort similarity desc, color asc 
+| limit 10
+| keep color, similarity
+;
+
+similarity:double
+0.978  
+;

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java

@@ -36,7 +36,12 @@ public class VectorSimilarityFunctionsIT extends AbstractEsqlIntegTestCase {
     public static Iterable<Object[]> parameters() throws Exception {
         List<Object[]> params = new ArrayList<>();
 
-        params.add(new Object[] { "v_cosine", VectorSimilarityFunction.COSINE });
+        if (EsqlCapabilities.Cap.COSINE_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            params.add(new Object[] { "v_cosine", VectorSimilarityFunction.COSINE });
+        }
+        if (EsqlCapabilities.Cap.DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            params.add(new Object[] { "v_dot_product", VectorSimilarityFunction.DOT_PRODUCT });
+        }
 
         return params;
     }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -1281,6 +1281,11 @@ public class EsqlCapabilities {
          */
         COSINE_VECTOR_SIMILARITY_FUNCTION(Build.current().isSnapshot()),
 
+        /**
+         * Dot product vector similarity function
+         */
+        DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION(Build.current().isSnapshot()),
+
         /**
          * Support for the options field of CATEGORIZE.
          */

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java

@@ -181,6 +181,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Trim;
 import org.elasticsearch.xpack.esql.expression.function.scalar.util.Delay;
 import org.elasticsearch.xpack.esql.expression.function.vector.CosineSimilarity;
+import org.elasticsearch.xpack.esql.expression.function.vector.DotProduct;
 import org.elasticsearch.xpack.esql.expression.function.vector.Knn;
 import org.elasticsearch.xpack.esql.parser.ParsingException;
 import org.elasticsearch.xpack.esql.session.Configuration;
@@ -491,7 +492,8 @@ public class EsqlFunctionRegistry {
                 def(StGeohex.class, StGeohex::new, "st_geohex"),
                 def(StGeohexToLong.class, StGeohexToLong::new, "st_geohex_to_long"),
                 def(StGeohexToString.class, StGeohexToString::new, "st_geohex_to_string"),
-                def(CosineSimilarity.class, CosineSimilarity::new, "v_cosine") } };
+                def(CosineSimilarity.class, CosineSimilarity::new, "v_cosine"),
+                def(DotProduct.class, DotProduct::new, "v_dot_product") } };
     }
 
     public EsqlFunctionRegistry snapshotRegistry() {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/DotProduct.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.vector;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo;
+import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+
+import java.io.IOException;
+
+import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;
+
+public class DotProduct extends VectorSimilarityFunction {
+
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        Expression.class,
+        "DotProduct",
+        DotProduct::new
+    );
+    static final SimilarityEvaluatorFunction SIMILARITY_FUNCTION = DOT_PRODUCT::compare;
+
+    @FunctionInfo(
+        returnType = "double",
+        preview = true,
+        description = "Calculates the dot product between two dense_vectors.",
+        examples = { @Example(file = "vector-dot-product", tag = "vector-dot-product") },
+        appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.DEVELOPMENT) }
+    )
+    public DotProduct(
+        Source source,
+        @Param(
+            name = "left",
+            type = { "dense_vector" },
+            description = "first dense_vector to calculate dot product similarity"
+        ) Expression left,
+        @Param(
+            name = "right",
+            type = { "dense_vector" },
+            description = "second dense_vector to calculate dot product similarity"
+        ) Expression right
+    ) {
+        super(source, left, right);
+    }
+
+    private DotProduct(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected BinaryScalarFunction replaceChildren(Expression newLeft, Expression newRight) {
+        return new DotProduct(source(), newLeft, newRight);
+    }
+
+    @Override
+    protected SimilarityEvaluatorFunction getSimilarityFunction() {
+        return SIMILARITY_FUNCTION;
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, DotProduct::new, left(), right());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java

@@ -33,6 +33,9 @@ public final class VectorWritables {
         if (EsqlCapabilities.Cap.COSINE_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
             entries.add(CosineSimilarity.ENTRY);
         }
+        if (EsqlCapabilities.Cap.DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            entries.add(DotProduct.ENTRY);
+        }
 
         return Collections.unmodifiableList(entries);
     }

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java

@@ -2362,6 +2362,13 @@ public class AnalyzerTests extends ESTestCase {
             checkDenseVectorImplicitCastingSimilarityFunction("v_cosine(vector, [0.342, 0.164, 0.234])", List.of(0.342f, 0.164f, 0.234f));
             checkDenseVectorImplicitCastingSimilarityFunction("v_cosine(vector, [1, 2, 3])", List.of(1f, 2f, 3f));
         }
+        if (EsqlCapabilities.Cap.DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            checkDenseVectorImplicitCastingSimilarityFunction(
+                "v_dot_product(vector, [0.342, 0.164, 0.234])",
+                List.of(0.342f, 0.164f, 0.234f)
+            );
+            checkDenseVectorImplicitCastingSimilarityFunction("v_dot_product(vector, [1, 2, 3])", List.of(1f, 2f, 3f));
+        }
     }
 
     private void checkDenseVectorImplicitCastingSimilarityFunction(String similarityFunction, List<Number> expectedElems) {
@@ -2385,6 +2392,9 @@ public class AnalyzerTests extends ESTestCase {
         if (EsqlCapabilities.Cap.COSINE_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
             checkNoDenseVectorFailsSimilarityFunction("v_cosine([0, 1, 2], 0.342)");
         }
+        if (EsqlCapabilities.Cap.DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            checkNoDenseVectorFailsSimilarityFunction("v_dot_product([0, 1, 2], 0.342)");
+        }
     }
 
     private void checkNoDenseVectorFailsSimilarityFunction(String similarityFunction) {

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java

@@ -2504,6 +2504,10 @@ public class VerifierTests extends ESTestCase {
             checkVectorSimilarityFunctionsNullArgs("v_cosine(null, vector)", "first");
             checkVectorSimilarityFunctionsNullArgs("v_cosine(vector, null)", "second");
         }
+        if (EsqlCapabilities.Cap.DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            checkVectorSimilarityFunctionsNullArgs("v_dot_product(null, vector)", "first");
+            checkVectorSimilarityFunctionsNullArgs("v_dot_product(vector, null)", "second");
+        }
     }
 
     private void checkVectorSimilarityFunctionsNullArgs(String functionInvocation, String argOrdinal) throws Exception {

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/vector/DotProductSimilarityTests.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.vector;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.FunctionName;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.List;
+import java.util.function.Supplier;
+
+@FunctionName("v_dot_product")
+public class DotProductSimilarityTests extends AbstractVectorSimilarityFunctionTestCase {
+
+    public DotProductSimilarityTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        super(testCaseSupplier);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        return similarityParameters(DotProduct.class.getSimpleName(), DotProduct.SIMILARITY_FUNCTION);
+    }
+
+    protected EsqlCapabilities.Cap capability() {
+        return EsqlCapabilities.Cap.DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION;
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new DotProduct(source, args.get(0), args.get(1));
+    }
+}